Add ExtractTarStep and SaveDockerStep

jib-core/src/main/java/com/google/cloud/tools/jib/blob/Blobs.java

@@ -18,11 +18,14 @@
 
 import com.google.cloud.tools.jib.hash.WritableContents;
 import com.google.cloud.tools.jib.json.JsonTemplate;
+import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
 
 /** Static methods for {@link Blob}. */
 public class Blobs {
@@ -77,5 +80,31 @@ public static byte[] writeToByteArray(Blob blob) throws IOException {
     return byteArrayOutputStream.toByteArray();
   }
 
+  /**
+   * Gets a {@link Blob} that is {@code blob} compressed.
+   *
+   * @param blob the {@link Blob} to compress
+   * @return the compressed {@link Blob}
+   */
+  public static Blob compress(Blob blob) {
+    return Blobs.from(
+        outputStream -> {
+          try (GZIPOutputStream compressorStream = new GZIPOutputStream(outputStream)) {
+            blob.writeTo(compressorStream);
+          }
+        });
+  }
+
+  /**
+   * Gets a {@link Blob} that is {@code blob} decompressed.
+   *
+   * @param blob the {@link Blob} to decompress
+   * @return the decompressed {@link Blob}
+   * @throws IOException if an I/O exception occurs
+   */
+  public static Blob decompress(Blob blob) throws IOException {
+    return Blobs.from(new GZIPInputStream(new ByteArrayInputStream(Blobs.writeToByteArray(blob))));
+  }
+
   private Blobs() {}
 }

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ExtractTarStep.java

@@ -0,0 +1,140 @@
+/*
+ * Copyright 2019 Google LLC.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.google.cloud.tools.jib.builder.steps;
+
+import com.fasterxml.jackson.databind.MapperFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.cloud.tools.jib.blob.Blob;
+import com.google.cloud.tools.jib.blob.BlobDescriptor;
+import com.google.cloud.tools.jib.blob.Blobs;
+import com.google.cloud.tools.jib.builder.steps.ExtractTarStep.LocalImage;
+import com.google.cloud.tools.jib.cache.CachedLayer;
+import com.google.cloud.tools.jib.docker.json.DockerManifestEntryTemplate;
+import com.google.cloud.tools.jib.image.Image;
+import com.google.cloud.tools.jib.image.LayerCountMismatchException;
+import com.google.cloud.tools.jib.image.json.BadContainerConfigurationFormatException;
+import com.google.cloud.tools.jib.image.json.ContainerConfigurationTemplate;
+import com.google.cloud.tools.jib.image.json.JsonToImageTranslator;
+import com.google.cloud.tools.jib.image.json.V22ManifestTemplate;
+import com.google.cloud.tools.jib.json.JsonTemplateMapper;
+import com.google.cloud.tools.jib.tar.TarExtractor;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.io.ByteStreams;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.zip.GZIPInputStream;
+
+/** Extracts a tar file base image. */
+public class ExtractTarStep implements Callable<LocalImage> {
+
+  static class LocalImage {
+    Image baseImage;
+    List<PreparedLayer> layers;
+
+    LocalImage(Image baseImage, List<PreparedLayer> layers) {
+      this.baseImage = baseImage;
+      this.layers = layers;
+    }
+  }
+
+  @VisibleForTesting
+  static boolean isGzipped(Path path) throws IOException {
+    try (InputStream inputStream = new BufferedInputStream(Files.newInputStream(path))) {
+      inputStream.mark(2);
+      int magic = (inputStream.read() & 0xff) | ((inputStream.read() << 8) & 0xff00);
+      return magic == GZIPInputStream.GZIP_MAGIC;
+    }
+  }
+
+  private final Path tarPath;
+  private final Path destination;
+
+  ExtractTarStep(Path tarPath, Path destination) {
+    this.tarPath = tarPath;
+    this.destination = destination;
+  }
+
+  @Override
+  public LocalImage call()
+      throws IOException, LayerCountMismatchException, BadContainerConfigurationFormatException {
+    TarExtractor.extract(tarPath, destination);
+    DockerManifestEntryTemplate loadManifest =
+        new ObjectMapper()
+            .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true)
+            .readValue(
+                Files.newInputStream(destination.resolve("manifest.json")),
+                DockerManifestEntryTemplate[].class)[0];
+    ContainerConfigurationTemplate configuration =
+        JsonTemplateMapper.readJsonFromFile(
+            destination.resolve(loadManifest.getConfig()), ContainerConfigurationTemplate.class);
+
+    List<String> layerFiles = loadManifest.getLayerFiles();
+    if (configuration.getLayerCount() != layerFiles.size()) {
+      throw new LayerCountMismatchException(
+          "Invalid base image format: manifest contains "
+              + layerFiles.size()
+              + " layers, but container configuration contains "
+              + configuration.getLayerCount()
+              + " layers");
+    }
+
+    // Check the first layer to see if the layers are compressed already. 'docker save' output is
+    // uncompressed, but a jib-built tar has compressed layers.
+    boolean layersAreCompressed =
+        layerFiles.size() > 0 && isGzipped(destination.resolve(layerFiles.get(0)));
+
+    // Process layer blobs
+    // TODO: Parallelize/other optimizations; calculating layer digests is slow for large layers
+    List<PreparedLayer> layers = new ArrayList<>();
+    V22ManifestTemplate newManifest = new V22ManifestTemplate();
+    for (int index = 0; index < layerFiles.size(); index++) {
+      Path file = destination.resolve(layerFiles.get(index));
+
+      // Compress layers if necessary and calculate the digest/size
+      Blob blob = layersAreCompressed ? Blobs.from(file) : Blobs.compress(Blobs.from(file));
+      BlobDescriptor blobDescriptor = blob.writeTo(ByteStreams.nullOutputStream());
+
+      // 'manifest' contains the layer files in the same order as the diff ids in 'configuration',
+      // so we don't need to recalculate those.
+      // https://containers.gitbook.io/build-containers-the-hard-way/#docker-load-format
+      CachedLayer layer =
+          CachedLayer.builder()
+              .setLayerBlob(blob)
+              .setLayerDigest(blobDescriptor.getDigest())
+              .setLayerSize(blobDescriptor.getSize())
+              .setLayerDiffId(configuration.getLayerDiffId(index))
+              .build();
+
+      // TODO: Check blob existence on target registry (online mode only)
+
+      layers.add(new PreparedLayer.Builder(layer).build());
+      newManifest.addLayer(blobDescriptor.getSize(), blobDescriptor.getDigest());
+    }
+
+    BlobDescriptor configDescriptor =
+        Blobs.from(configuration).writeTo(ByteStreams.nullOutputStream());
+    newManifest.setContainerConfiguration(configDescriptor.getSize(), configDescriptor.getDigest());
+    Image image = JsonToImageTranslator.toImage(newManifest, configuration);
+    return new LocalImage(image, layers);
+  }
+}

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/SaveDockerStep.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2019 Google LLC.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.google.cloud.tools.jib.builder.steps;
+
+import com.google.cloud.tools.jib.api.ImageReference;
+import com.google.cloud.tools.jib.configuration.BuildConfiguration;
+import com.google.cloud.tools.jib.docker.DockerClient;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.Callable;
+
+/** Saves an image from the docker daemon. */
+public class SaveDockerStep implements Callable<Path> {
+
+  private final BuildConfiguration buildConfiguration;
+  private final DockerClient dockerClient;
+
+  SaveDockerStep(BuildConfiguration buildConfiguration, DockerClient dockerClient) {
+    this.buildConfiguration = buildConfiguration;
+    this.dockerClient = dockerClient;
+  }
+
+  @Override
+  public Path call() throws IOException, InterruptedException {
+    Path outputPath = Files.createTempDirectory("jib-docker-save").resolve("out.tar");
+    ImageReference imageReference = buildConfiguration.getBaseImageConfiguration().getImage();
+    dockerClient.save(imageReference, outputPath);
+    return outputPath;
+  }
+}

jib-core/src/main/java/com/google/cloud/tools/jib/cache/CachedLayer.java

@@ -27,7 +27,7 @@
 public class CachedLayer implements Layer {
 
   /** Builds a {@link CachedLayer}. */
-  static class Builder {
+  public static class Builder {
 
     @Nullable private DescriptorDigest layerDigest;
     @Nullable private DescriptorDigest layerDiffId;
@@ -36,22 +36,22 @@ static class Builder {
 
     private Builder() {}
 
-    Builder setLayerDigest(DescriptorDigest layerDigest) {
+    public Builder setLayerDigest(DescriptorDigest layerDigest) {
       this.layerDigest = layerDigest;
       return this;
     }
 
-    Builder setLayerDiffId(DescriptorDigest layerDiffId) {
+    public Builder setLayerDiffId(DescriptorDigest layerDiffId) {
       this.layerDiffId = layerDiffId;
       return this;
     }
 
-    Builder setLayerSize(long layerSize) {
+    public Builder setLayerSize(long layerSize) {
       this.layerSize = layerSize;
       return this;
     }
 
-    Builder setLayerBlob(Blob layerBlob) {
+    public Builder setLayerBlob(Blob layerBlob) {
       this.layerBlob = layerBlob;
       return this;
     }
@@ -60,7 +60,7 @@ boolean hasLayerBlob() {
       return layerBlob != null;
     }
 
-    CachedLayer build() {
+    public CachedLayer build() {
       return new CachedLayer(
           Preconditions.checkNotNull(layerDigest, "layerDigest required"),
           Preconditions.checkNotNull(layerDiffId, "layerDiffId required"),
@@ -74,7 +74,7 @@ CachedLayer build() {
    *
    * @return the new {@link Builder}
    */
-  static Builder builder() {
+  public static Builder builder() {
     return new Builder();
   }
 

jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ContainerConfigurationTemplate.java

@@ -19,7 +19,6 @@
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.google.cloud.tools.jib.api.DescriptorDigest;
 import com.google.cloud.tools.jib.json.JsonTemplate;
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import java.util.ArrayList;
 import java.util.List;
@@ -369,8 +368,11 @@ String getContainerUser() {
     return config.Volumes;
   }
 
-  @VisibleForTesting
-  DescriptorDigest getLayerDiffId(int index) {
+  public DescriptorDigest getLayerDiffId(int index) {
     return rootfs.diff_ids.get(index);
   }
+
+  public int getLayerCount() {
+    return rootfs.diff_ids.size();
+  }
 }

jib-core/src/main/java/com/google/cloud/tools/jib/json/JsonTemplateMapper.java

@@ -16,8 +16,6 @@
 
 package com.google.cloud.tools.jib.json;
 
-import com.fasterxml.jackson.core.JsonGenerationException;
-import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.type.CollectionType;
 import java.io.ByteArrayOutputStream;
@@ -134,50 +132,42 @@ public static <T extends JsonTemplate> List<T> readListOfJson(
     return objectMapper.readValue(jsonString, listType);
   }
 
-  public static String toUtf8String(JsonTemplate template)
-      throws JsonGenerationException, JsonMappingException, IOException {
+  public static String toUtf8String(JsonTemplate template) throws IOException {
     return toUtf8String((Object) template);
   }
 
-  public static String toUtf8String(List<? extends JsonTemplate> templates)
-      throws JsonGenerationException, JsonMappingException, IOException {
+  public static String toUtf8String(List<? extends JsonTemplate> templates) throws IOException {
     return toUtf8String((Object) templates);
   }
 
-  public static byte[] toByteArray(JsonTemplate template)
-      throws JsonGenerationException, JsonMappingException, IOException {
+  public static byte[] toByteArray(JsonTemplate template) throws IOException {
     return toByteArray((Object) template);
   }
 
-  public static byte[] toByteArray(List<? extends JsonTemplate> templates)
-      throws JsonGenerationException, JsonMappingException, IOException {
+  public static byte[] toByteArray(List<? extends JsonTemplate> templates) throws IOException {
     return toByteArray((Object) templates);
   }
 
-  public static void writeTo(JsonTemplate template, OutputStream out)
-      throws JsonGenerationException, JsonMappingException, IOException {
+  public static void writeTo(JsonTemplate template, OutputStream out) throws IOException {
     writeTo((Object) template, out);
   }
 
   public static void writeTo(List<? extends JsonTemplate> templates, OutputStream out)
-      throws JsonGenerationException, JsonMappingException, IOException {
+      throws IOException {
     writeTo((Object) templates, out);
   }
 
-  private static String toUtf8String(Object template)
-      throws JsonGenerationException, JsonMappingException, IOException {
+  private static String toUtf8String(Object template) throws IOException {
     return new String(toByteArray(template), StandardCharsets.UTF_8);
   }
 
-  private static byte[] toByteArray(Object template)
-      throws JsonGenerationException, JsonMappingException, IOException {
+  private static byte[] toByteArray(Object template) throws IOException {
     ByteArrayOutputStream out = new ByteArrayOutputStream();
     writeTo(template, out);
     return out.toByteArray();
   }
 
-  private static void writeTo(Object template, OutputStream out)
-      throws JsonGenerationException, JsonMappingException, IOException {
+  private static void writeTo(Object template, OutputStream out) throws IOException {
     objectMapper.writeValue(out, template);
   }