diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java index 775e7ba787..ee0bc497a6 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java @@ -18,12 +18,15 @@ import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.cloud.tools.jib.api.DescriptorDigest; import com.google.cloud.tools.jib.blob.Blob; import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.blob.Blobs; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.builder.steps.ExtractTarStep.LocalImage; +import com.google.cloud.tools.jib.cache.Cache; +import com.google.cloud.tools.jib.cache.CacheCorruptedException; import com.google.cloud.tools.jib.cache.CachedLayer; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.docker.json.DockerManifestEntryTemplate; @@ -46,6 +49,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.concurrent.Callable; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; @@ -96,7 +100,8 @@ static boolean isGzipped(Path path) throws IOException { @Override public LocalImage call() - throws IOException, LayerCountMismatchException, BadContainerConfigurationFormatException { + throws IOException, LayerCountMismatchException, BadContainerConfigurationFormatException, + CacheCorruptedException { Path destination = Files.createTempDirectory("jib-extract-tar"); try (TimerEventDispatcher ignored = new TimerEventDispatcher( @@ -132,56 +137,23 @@ public LocalImage call() // Process layer blobs // TODO: Optimize; compressing/calculating layer digests is slow - // e.g. parallelize, cache layers, faster compression method + // e.g. parallelize, faster compression method try (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.create( "processing base image layers", layerFiles.size())) { List layers = new ArrayList<>(layerFiles.size()); V22ManifestTemplate v22Manifest = new V22ManifestTemplate(); - List childProgressFactories = new ArrayList<>(); - for (String ignored1 : layerFiles) { - childProgressFactories.add(progressEventDispatcher.newChildProducer()); - } - for (int index = 0; index < layerFiles.size(); index++) { - Path file = destination.resolve(layerFiles.get(index)); - - // Compress layers if necessary and calculate the digest/size - Blob blob = Blobs.from(file); - try (ProgressEventDispatcher childDispatcher = - childProgressFactories - .get(index) - .create("compressing " + file, Files.size(file)); - ThrottledAccumulatingConsumer throttledProgressReporter = - new ThrottledAccumulatingConsumer(childDispatcher::dispatchProgress)) { - if (!layersAreCompressed) { - Path compressedFile = destination.resolve(layerFiles.get(index) + ".compressed"); - try (GZIPOutputStream compressorStream = - new GZIPOutputStream(Files.newOutputStream(compressedFile)); - NotifyingOutputStream notifyingOutputStream = - new NotifyingOutputStream(compressorStream, throttledProgressReporter)) { - blob.writeTo(notifyingOutputStream); - } - blob = Blobs.from(compressedFile); - } - } - BlobDescriptor blobDescriptor = blob.writeTo(ByteStreams.nullOutputStream()); - - // 'manifest' contains the layer files in the same order as the diff ids in - // 'configuration', so we don't need to recalculate those. - // https://containers.gitbook.io/build-containers-the-hard-way/#docker-load-format + Path layerFile = destination.resolve(layerFiles.get(index)); CachedLayer layer = - CachedLayer.builder() - .setLayerBlob(blob) - .setLayerDigest(blobDescriptor.getDigest()) - .setLayerSize(blobDescriptor.getSize()) - .setLayerDiffId(configurationTemplate.getLayerDiffId(index)) - .build(); - + getCachedTarLayer( + configurationTemplate.getLayerDiffId(index), + layerFile, + layersAreCompressed, + progressEventDispatcher.newChildProducer()); layers.add(new PreparedLayer.Builder(layer).build()); - v22Manifest.addLayer(blobDescriptor.getSize(), blobDescriptor.getDigest()); - progressEventDispatcher.dispatchProgress(1); + v22Manifest.addLayer(layer.getSize(), layer.getDigest()); } BlobDescriptor configDescriptor = @@ -193,4 +165,42 @@ public LocalImage call() } } } + + private CachedLayer getCachedTarLayer( + DescriptorDigest diffId, + Path layerFile, + boolean layersAreCompressed, + ProgressEventDispatcher.Factory progressEventDispatcherFactory) + throws IOException, CacheCorruptedException { + try (ProgressEventDispatcher childDispatcher = + progressEventDispatcherFactory.create( + "compressing layer " + diffId, Files.size(layerFile)); + ThrottledAccumulatingConsumer throttledProgressReporter = + new ThrottledAccumulatingConsumer(childDispatcher::dispatchProgress)) { + Cache cache = buildConfiguration.getBaseImageLayersCache(); + + // Retrieve pre-compressed layer from cache + Optional optionalLayer = cache.retrieveTarLayer(diffId); + if (optionalLayer.isPresent()) { + return optionalLayer.get(); + } + + // Just write layers that are already compressed + if (layersAreCompressed) { + return cache.writeTarLayer(diffId, Blobs.from(layerFile)); + } + + // Compress uncompressed layers while writing + Blob compressedBlob = + Blobs.from( + outputStream -> { + try (GZIPOutputStream compressorStream = new GZIPOutputStream(outputStream); + NotifyingOutputStream notifyingOutputStream = + new NotifyingOutputStream(compressorStream, throttledProgressReporter)) { + Blobs.from(layerFile).writeTo(notifyingOutputStream); + } + }); + return cache.writeTarLayer(diffId, compressedBlob); + } + } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/Cache.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/Cache.java index 6a5c946eed..6bd8532448 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/Cache.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/Cache.java @@ -116,6 +116,20 @@ public CachedLayer writeUncompressedLayer( uncompressedLayerBlob, LayerEntriesSelector.generateSelector(layerEntries)); } + /** + * Caches a layer that was extracted from a local base image, and names the file using the + * provided diff id. + * + * @param diffId the diff id + * @param compressedBlob the compressed layer blob + * @return the {@link CachedLayer} for the written layer + * @throws IOException if an I/O exception occurs + */ + public CachedLayer writeTarLayer(DescriptorDigest diffId, Blob compressedBlob) + throws IOException { + return cacheStorageWriter.writeTarLayer(diffId, compressedBlob); + } + /** * Retrieves the cached manifest and container configuration for an image reference. * @@ -160,4 +174,17 @@ public Optional retrieve(DescriptorDigest layerDigest) throws IOException, CacheCorruptedException { return cacheStorageReader.retrieve(layerDigest); } + + /** + * Retrieves a {@link CachedLayer} for a local base image layer with the given diff id. + * + * @param diffId the diff id + * @return the {@link CachedLayer} with the given diff id + * @throws CacheCorruptedException if the cache was found to be corrupted + * @throws IOException if an I/O exception occurs + */ + public Optional retrieveTarLayer(DescriptorDigest diffId) + throws IOException, CacheCorruptedException { + return cacheStorageReader.retrieveTarLayer(diffId); + } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageFiles.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageFiles.java index 3296a71da3..e6aa3d3661 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageFiles.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageFiles.java @@ -26,6 +26,7 @@ class CacheStorageFiles { private static final String LAYERS_DIRECTORY = "layers"; + private static final String LOCAL_DIRECTORY = "local"; private static final String IMAGES_DIRECTORY = "images"; private static final String SELECTORS_DIRECTORY = "selectors"; private static final String TEMPORARY_DIRECTORY = "tmp"; @@ -54,14 +55,14 @@ static boolean isLayerFile(Path file) { * @return the diff ID portion of the layer file filename * @throws CacheCorruptedException if no valid diff ID could be parsed */ - DescriptorDigest getDiffId(Path layerFile) throws CacheCorruptedException { + DescriptorDigest getDigestFromFilename(Path layerFile) throws CacheCorruptedException { try { - String diffId = layerFile.getFileName().toString(); - return DescriptorDigest.fromHash(diffId); + String hash = layerFile.getFileName().toString(); + return DescriptorDigest.fromHash(hash); } catch (DigestException | IndexOutOfBoundsException ex) { throw new CacheCorruptedException( - cacheDirectory, "Layer file did not include valid diff ID: " + layerFile, ex); + cacheDirectory, "Layer file did not include valid hash: " + layerFile, ex); } } @@ -125,6 +126,15 @@ Path getLayerDirectory(DescriptorDigest layerDigest) { return getLayersDirectory().resolve(layerDigest.getHash()); } + /** + * Resolves the {@link #LOCAL_DIRECTORY} in the {@link #cacheDirectory}. + * + * @return the directory containing local base image layers + */ + Path getLocalDirectory() { + return cacheDirectory.resolve(LOCAL_DIRECTORY); + } + /** * Gets the directory to store the image manifest and configuration. * diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageReader.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageReader.java index b9b4488836..3fbd217ebf 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageReader.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageReader.java @@ -168,7 +168,7 @@ Optional retrieve(DescriptorDigest layerDigest) if (layerFiles.size() != 1) { throw new CacheCorruptedException( cacheStorageFiles.getCacheDirectory(), - "No or multiple layer files found for layer with digest " + "No or multiple layer files found for layer hash " + layerDigest.getHash() + " in directory: " + layerDirectory); @@ -180,7 +180,45 @@ Optional retrieve(DescriptorDigest layerDigest) .setLayerDigest(layerDigest) .setLayerSize(Files.size(layerFile)) .setLayerBlob(Blobs.from(layerFile)) - .setLayerDiffId(cacheStorageFiles.getDiffId(layerFile)) + .setLayerDiffId(cacheStorageFiles.getDigestFromFilename(layerFile)) + .build()); + } + } + + /** + * Retrieves the {@link CachedLayer} for the local base image layer with the given diff ID. + * + * @param diffId the diff ID + * @return the {@link CachedLayer} referenced by the diff ID, if found + * @throws CacheCorruptedException if the cache was found to be corrupted + * @throws IOException if an I/O exception occurs + */ + Optional retrieveTarLayer(DescriptorDigest diffId) + throws IOException, CacheCorruptedException { + Path layerDirectory = cacheStorageFiles.getLocalDirectory().resolve(diffId.getHash()); + if (!Files.exists(layerDirectory)) { + return Optional.empty(); + } + + try (Stream files = Files.list(layerDirectory)) { + List layerFiles = + files.filter(CacheStorageFiles::isLayerFile).collect(Collectors.toList()); + if (layerFiles.size() != 1) { + throw new CacheCorruptedException( + cacheStorageFiles.getCacheDirectory(), + "No or multiple layer files found for layer hash " + + diffId.getHash() + + " in directory: " + + layerDirectory); + } + + Path layerFile = layerFiles.get(0); + return Optional.of( + CachedLayer.builder() + .setLayerDigest(cacheStorageFiles.getDigestFromFilename(layerFile)) + .setLayerSize(Files.size(layerFile)) + .setLayerBlob(Blobs.from(layerFile)) + .setLayerDiffId(diffId) .build()); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java index ed2f6ac7b8..5704ef4434 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheStorageWriter.java @@ -251,6 +251,47 @@ CachedLayer writeUncompressed(Blob uncompressedLayerBlob, @Nullable DescriptorDi } } + /** + * Saves a local base image layer. + * + * @param diffId the layer blob's diff ID + * @param compressedBlob the blob to save + * @throws IOException if an I/O exception occurs + */ + CachedLayer writeTarLayer(DescriptorDigest diffId, Blob compressedBlob) throws IOException { + Files.createDirectories(cacheStorageFiles.getLocalDirectory()); + Files.createDirectories(cacheStorageFiles.getTemporaryDirectory()); + try (TemporaryDirectory temporaryDirectory = + new TemporaryDirectory(cacheStorageFiles.getTemporaryDirectory())) { + Path temporaryLayerDirectory = temporaryDirectory.getDirectory(); + Path temporaryLayerFile = cacheStorageFiles.getTemporaryLayerFile(temporaryLayerDirectory); + + BlobDescriptor layerBlobDescriptor; + try (OutputStream fileOutputStream = + new BufferedOutputStream(Files.newOutputStream(temporaryLayerFile))) { + layerBlobDescriptor = compressedBlob.writeTo(fileOutputStream); + } + + // Renames the temporary layer file to its digest + // (temp/temp -> temp/) + String fileName = layerBlobDescriptor.getDigest().getHash(); + Path digestLayerFile = temporaryLayerDirectory.resolve(fileName); + moveIfDoesNotExist(temporaryLayerFile, digestLayerFile); + + // Moves the temporary directory to directory named with diff ID + // (temp/ -> /) + Path destination = cacheStorageFiles.getLocalDirectory().resolve(diffId.getHash()); + moveIfDoesNotExist(temporaryLayerDirectory, destination); + + return CachedLayer.builder() + .setLayerDigest(layerBlobDescriptor.getDigest()) + .setLayerDiffId(diffId) + .setLayerSize(layerBlobDescriptor.getSize()) + .setLayerBlob(Blobs.from(destination.resolve(fileName))) + .build(); + } + } + /** * Saves the manifest and container configuration for a V2.2 or OCI image. * @@ -286,7 +327,7 @@ void writeMetadata(ImageReference imageReference, V21ManifestTemplate manifestTe Path imageDirectory = cacheStorageFiles.getImageDirectory(imageReference); Files.createDirectories(imageDirectory); - try (LockFile ignored1 = LockFile.lock(imageDirectory.resolve("lock"))) { + try (LockFile ignored = LockFile.lock(imageDirectory.resolve("lock"))) { writeMetadata(manifestTemplate, imageDirectory.resolve("manifest.json")); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java index 729806da5a..4afc6f09f5 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStepTest.java @@ -18,6 +18,8 @@ import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.steps.ExtractTarStep.LocalImage; +import com.google.cloud.tools.jib.cache.Cache; +import com.google.cloud.tools.jib.cache.CacheCorruptedException; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.event.EventHandlers; import com.google.cloud.tools.jib.image.LayerCountMismatchException; @@ -54,7 +56,9 @@ private static Path getResource(String resource) throws URISyntaxException { } @Before - public void setup() { + public void setup() throws IOException { + Mockito.when(buildConfiguration.getBaseImageLayersCache()) + .thenReturn(Cache.withDirectory(temporaryFolder.newFolder().toPath())); Mockito.when(buildConfiguration.getEventHandlers()).thenReturn(eventHandlers); Mockito.when(progressEventDispatcherFactory.create(Mockito.anyString(), Mockito.anyLong())) .thenReturn(progressEventDispatcher); @@ -66,7 +70,7 @@ public void setup() { @Test public void testCall_validDocker() throws URISyntaxException, LayerCountMismatchException, - BadContainerConfigurationFormatException, IOException { + BadContainerConfigurationFormatException, IOException, CacheCorruptedException { Path dockerBuild = getResource("core/extraction/docker-save.tar"); LocalImage result = new ExtractTarStep(buildConfiguration, dockerBuild, progressEventDispatcherFactory).call(); @@ -91,7 +95,7 @@ public void testCall_validDocker() @Test public void testCall_validTar() throws URISyntaxException, LayerCountMismatchException, - BadContainerConfigurationFormatException, IOException { + BadContainerConfigurationFormatException, IOException, CacheCorruptedException { Path tarBuild = getResource("core/extraction/jib-image.tar"); LocalImage result = new ExtractTarStep(buildConfiguration, tarBuild, progressEventDispatcherFactory).call(); diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageFilesTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageFilesTest.java index bf08828f55..fe217f46d7 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageFilesTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageFilesTest.java @@ -48,7 +48,7 @@ public void testGetDiffId() throws DigestException, CacheCorruptedException { Assert.assertEquals( DescriptorDigest.fromHash( "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"), - TEST_CACHE_STORAGE_FILES.getDiffId( + TEST_CACHE_STORAGE_FILES.getDigestFromFilename( Paths.get( "layer", "file", @@ -56,25 +56,25 @@ public void testGetDiffId() throws DigestException, CacheCorruptedException { Assert.assertEquals( DescriptorDigest.fromHash( "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), - TEST_CACHE_STORAGE_FILES.getDiffId( + TEST_CACHE_STORAGE_FILES.getDigestFromFilename( Paths.get("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))); } @Test public void testGetDiffId_corrupted() { try { - TEST_CACHE_STORAGE_FILES.getDiffId(Paths.get("not long enough")); + TEST_CACHE_STORAGE_FILES.getDigestFromFilename(Paths.get("not long enough")); Assert.fail("Should have thrown CacheCorruptedException"); } catch (CacheCorruptedException ex) { Assert.assertThat( ex.getMessage(), - CoreMatchers.startsWith("Layer file did not include valid diff ID: not long enough")); + CoreMatchers.startsWith("Layer file did not include valid hash: not long enough")); Assert.assertThat(ex.getCause(), CoreMatchers.instanceOf(DigestException.class)); } try { - TEST_CACHE_STORAGE_FILES.getDiffId( + TEST_CACHE_STORAGE_FILES.getDigestFromFilename( Paths.get( "not valid hash bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")); Assert.fail("Should have thrown CacheCorruptedException"); @@ -83,7 +83,7 @@ public void testGetDiffId_corrupted() { Assert.assertThat( ex.getMessage(), CoreMatchers.startsWith( - "Layer file did not include valid diff ID: " + "Layer file did not include valid hash: " + "not valid hash bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")); Assert.assertThat(ex.getCause(), CoreMatchers.instanceOf(DigestException.class)); } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java index 315358f39f..50fc375f25 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java @@ -198,13 +198,56 @@ public void testRetrieve() throws IOException, CacheCorruptedException { Assert.assertThat( ex.getMessage(), CoreMatchers.startsWith( - "No or multiple layer files found for layer with digest " + "No or multiple layer files found for layer hash " + layerDigest.getHash() + " in directory: " + cacheStorageFiles.getLayerDirectory(layerDigest))); } } + @Test + public void testRetrieveTarLayer() throws IOException, CacheCorruptedException { + CacheStorageFiles cacheStorageFiles = + new CacheStorageFiles(temporaryFolder.newFolder().toPath()); + + CacheStorageReader cacheStorageReader = new CacheStorageReader(cacheStorageFiles); + + // Creates the test layer directory. + Path localDirectory = cacheStorageFiles.getLocalDirectory(); + DescriptorDigest layerDigest = layerDigest1; + DescriptorDigest layerDiffId = layerDigest2; + Files.createDirectories(localDirectory.resolve(layerDiffId.getHash())); + try (OutputStream out = + Files.newOutputStream( + localDirectory.resolve(layerDiffId.getHash()).resolve(layerDigest.getHash()))) { + out.write("layerBlob".getBytes(StandardCharsets.UTF_8)); + } + + // Checks that the CachedLayer is retrieved correctly. + Optional optionalCachedLayer = cacheStorageReader.retrieveTarLayer(layerDiffId); + Assert.assertTrue(optionalCachedLayer.isPresent()); + Assert.assertEquals(layerDigest, optionalCachedLayer.get().getDigest()); + Assert.assertEquals(layerDiffId, optionalCachedLayer.get().getDiffId()); + Assert.assertEquals("layerBlob".length(), optionalCachedLayer.get().getSize()); + Assert.assertEquals("layerBlob", Blobs.writeToString(optionalCachedLayer.get().getBlob())); + + // Checks that multiple layer files means the cache is corrupted. + Files.createFile(localDirectory.resolve(layerDiffId.getHash()).resolve(layerDiffId.getHash())); + try { + cacheStorageReader.retrieveTarLayer(layerDiffId); + Assert.fail("Should have thrown CacheCorruptedException"); + + } catch (CacheCorruptedException ex) { + Assert.assertThat( + ex.getMessage(), + CoreMatchers.startsWith( + "No or multiple layer files found for layer hash " + + layerDiffId.getHash() + + " in directory: " + + localDirectory.resolve(layerDiffId.getHash()))); + } + } + @Test public void testSelect_invalidLayerDigest() throws IOException { CacheStorageFiles cacheStorageFiles = diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java index ecf48e2f40..1a24ab542f 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java @@ -74,7 +74,7 @@ public void setUp() throws IOException { } @Test - public void testWrite_compressed() throws IOException { + public void testWriteCompressed() throws IOException { Blob uncompressedLayerBlob = Blobs.from("uncompressedLayerBlob"); CachedLayer cachedLayer = @@ -84,7 +84,7 @@ public void testWrite_compressed() throws IOException { } @Test - public void testWrite_uncompressed() throws IOException { + public void testWriteUncompressed() throws IOException { Blob uncompressedLayerBlob = Blobs.from("uncompressedLayerBlob"); DescriptorDigest layerDigest = getDigest(compress(uncompressedLayerBlob)).getDigest(); DescriptorDigest selector = getDigest(Blobs.from("selector")).getDigest(); @@ -101,6 +101,34 @@ public void testWrite_uncompressed() throws IOException { Assert.assertEquals(layerDigest.getHash(), Blobs.writeToString(Blobs.from(selectorFile))); } + @Test + public void testWriteTarLayer() throws IOException { + Blob uncompressedLayerBlob = Blobs.from("uncompressedLayerBlob"); + DescriptorDigest diffId = getDigest(uncompressedLayerBlob).getDigest(); + + CachedLayer cachedLayer = + new CacheStorageWriter(cacheStorageFiles) + .writeTarLayer(diffId, compress(uncompressedLayerBlob)); + + BlobDescriptor layerBlobDescriptor = getDigest(compress(uncompressedLayerBlob)); + + // Verifies cachedLayer is correct. + Assert.assertEquals(layerBlobDescriptor.getDigest(), cachedLayer.getDigest()); + Assert.assertEquals(diffId, cachedLayer.getDiffId()); + Assert.assertEquals(layerBlobDescriptor.getSize(), cachedLayer.getSize()); + Assert.assertArrayEquals( + Blobs.writeToByteArray(uncompressedLayerBlob), + Blobs.writeToByteArray(decompress(cachedLayer.getBlob()))); + + // Verifies that the files are present. + Assert.assertTrue( + Files.exists( + cacheStorageFiles + .getLocalDirectory() + .resolve(cachedLayer.getDiffId().getHash()) + .resolve(cachedLayer.getDigest().getHash()))); + } + @Test public void testWriteMetadata_v21() throws IOException, URISyntaxException, InvalidImageReferenceException { diff --git a/jib-maven-plugin/src/integration-test/java/com/google/cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java b/jib-maven-plugin/src/integration-test/java/com/google/cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java index 9764e7cec4..fa16447c84 100644 --- a/jib-maven-plugin/src/integration-test/java/com/google/cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java +++ b/jib-maven-plugin/src/integration-test/java/com/google/cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java @@ -159,7 +159,7 @@ private static String buildAndRun( } private static String buildAndRunFromLocalBase( - Path projectRoot, String targetImage, String baseImage) + Path projectRoot, String targetImage, String baseImage, boolean buildTwice) throws VerificationException, IOException, InterruptedException { Verifier verifier = new Verifier(projectRoot.toString()); verifier.setSystemProperty("jib.useOnlyProjectCache", "true"); @@ -170,7 +170,20 @@ private static String buildAndRunFromLocalBase( verifier.addCliOption("-X"); verifier.addCliOption("--file=pom-localbase.xml"); verifier.executeGoals(Arrays.asList("clean", "compile")); + if (!buildTwice) { + verifier.executeGoal("jib:build"); + return pullAndRunBuiltImage(targetImage); + } + verifier.executeGoal("jib:build"); + float timeOne = getBuildTimeFromVerifierLog(verifier); + + verifier.resetStreams(); + verifier.executeGoal("jib:build"); + float timeTwo = getBuildTimeFromVerifierLog(verifier); + + String failMessage = "First build time (%s) is not greater than second build time (%s)"; + Assert.assertTrue(String.format(failMessage, timeOne, timeTwo), timeOne > timeTwo); return pullAndRunBuiltImage(targetImage); } @@ -420,7 +433,8 @@ public void testBuild_dockerDaemonBase() buildAndRunFromLocalBase( simpleTestProject.getProjectRoot(), targetImage, - "docker://gcr.io/distroless/java:latest")); + "docker://gcr.io/distroless/java:latest", + false)); } @Test @@ -435,7 +449,8 @@ public void testBuild_tarBase() throws IOException, InterruptedException, Verifi Assert.assertEquals( "Hello, world. An argument.\n1970-01-01T00:00:01Z\nrw-r--r--\nrw-r--r--\nfoo\ncat\n" + "1970-01-01T00:00:01Z\n1970-01-01T00:00:01Z\n", - buildAndRunFromLocalBase(simpleTestProject.getProjectRoot(), targetImage, "tar://" + path)); + buildAndRunFromLocalBase( + simpleTestProject.getProjectRoot(), targetImage, "tar://" + path, true)); } @Test