Implements Proposal: Address Base Image Reproducibility

jib-core/src/main/java/com/google/cloud/tools/jib/builder/BuildAndCacheApplicationLayersStep.java

@@ -21,7 +21,7 @@
 import com.google.cloud.tools.jib.cache.CacheReader;
 import com.google.cloud.tools.jib.cache.CacheWriter;
 import com.google.cloud.tools.jib.cache.CachedLayer;
-import com.google.cloud.tools.jib.image.LayerBuilder;
+import com.google.cloud.tools.jib.image.ReproducibleLayerBuilder;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningExecutorService;
 import java.nio.file.Path;
@@ -89,11 +89,10 @@ private ListenableFuture<CachedLayer> buildAndCacheLayerAsync(
               return cachedLayer;
             }
 
-            LayerBuilder layerBuilder =
-                new LayerBuilder(
-                    sourceFiles, extractionPath, buildConfiguration.getEnableReproducibleBuilds());
+            ReproducibleLayerBuilder reproducibleLayerBuilder =
+                new ReproducibleLayerBuilder(sourceFiles, extractionPath);
 
-            cachedLayer = new CacheWriter(cache).writeLayer(layerBuilder);
+            cachedLayer = new CacheWriter(cache).writeLayer(reproducibleLayerBuilder);
             // TODO: Remove
             buildConfiguration
                 .getBuildLogger()

jib-core/src/main/java/com/google/cloud/tools/jib/builder/BuildConfiguration.java

@@ -39,7 +39,6 @@ public static class Builder {
     @Nullable private ImageReference targetImageReference;
     private List<String> credentialHelperNames = new ArrayList<>();
     private RegistryCredentials knownRegistryCredentials = RegistryCredentials.none();
-    private boolean enableReproducibleBuilds = true;
     @Nullable private String mainClass;
     private List<String> jvmFlags = new ArrayList<>();
     private Map<String, String> environmentMap = new HashMap<>();
@@ -76,11 +75,6 @@ public Builder setKnownRegistryCredentials(
       return this;
     }
 
-    public Builder setEnableReproducibleBuilds(boolean isEnabled) {
-      enableReproducibleBuilds = isEnabled;
-      return this;
-    }
-
     public Builder setMainClass(@Nullable String mainClass) {
       this.mainClass = mainClass;
       return this;
@@ -130,7 +124,6 @@ public BuildConfiguration build() {
               targetImageReference,
               credentialHelperNames,
               knownRegistryCredentials,
-              enableReproducibleBuilds,
               mainClass,
               jvmFlags,
               environmentMap,
@@ -177,7 +170,6 @@ public static boolean isValidJavaClass(String className) {
   private ImageReference targetImageReference;
   private List<String> credentialHelperNames;
   private RegistryCredentials knownRegistryCredentials;
-  private boolean enableReproducibleBuilds;
   private String mainClass;
   private List<String> jvmFlags;
   private Map<String, String> environmentMap;
@@ -194,7 +186,6 @@ private BuildConfiguration(
       ImageReference targetImageReference,
       List<String> credentialHelperNames,
       RegistryCredentials knownRegistryCredentials,
-      boolean enableReproducibleBuilds,
       String mainClass,
       List<String> jvmFlags,
       Map<String, String> environmentMap,
@@ -204,7 +195,6 @@ private BuildConfiguration(
     this.targetImageReference = targetImageReference;
     this.credentialHelperNames = Collections.unmodifiableList(credentialHelperNames);
     this.knownRegistryCredentials = knownRegistryCredentials;
-    this.enableReproducibleBuilds = enableReproducibleBuilds;
     this.mainClass = mainClass;
     this.jvmFlags = Collections.unmodifiableList(jvmFlags);
     this.environmentMap = Collections.unmodifiableMap(environmentMap);
@@ -247,10 +237,6 @@ public List<String> getCredentialHelperNames() {
     return credentialHelperNames;
   }
 
-  public boolean getEnableReproducibleBuilds() {
-    return enableReproducibleBuilds;
-  }
-
   public String getMainClass() {
     return mainClass;
   }

jib-core/src/main/java/com/google/cloud/tools/jib/cache/CacheWriter.java

@@ -19,8 +19,8 @@
 import com.google.cloud.tools.jib.blob.BlobDescriptor;
 import com.google.cloud.tools.jib.hash.CountingDigestOutputStream;
 import com.google.cloud.tools.jib.image.DescriptorDigest;
-import com.google.cloud.tools.jib.image.LayerBuilder;
 import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException;
+import com.google.cloud.tools.jib.image.ReproducibleLayerBuilder;
 import com.google.cloud.tools.jib.image.UnwrittenLayer;
 import com.google.common.io.ByteStreams;
 import com.google.common.io.CountingOutputStream;
@@ -46,15 +46,15 @@ public CacheWriter(Cache cache) {
   }
 
   /**
-   * Builds an {@link UnwrittenLayer} from a {@link LayerBuilder} and compresses and writes the
-   * {@link UnwrittenLayer}'s uncompressed layer content BLOB to cache.
+   * Builds an {@link UnwrittenLayer} from a {@link ReproducibleLayerBuilder} and compresses and
+   * writes the {@link UnwrittenLayer}'s uncompressed layer content BLOB to cache.
    *
-   * @param layerBuilder the layer builder
+   * @param reproducibleLayerBuilder the layer builder
    * @return the cached layer
    */
-  public CachedLayer writeLayer(LayerBuilder layerBuilder)
+  public CachedLayer writeLayer(ReproducibleLayerBuilder reproducibleLayerBuilder)
       throws IOException, LayerPropertyNotFoundException {
-    UnwrittenLayer unwrittenLayer = layerBuilder.build();
+    UnwrittenLayer unwrittenLayer = reproducibleLayerBuilder.build();
 
     // Writes to a temporary file first because the UnwrittenLayer needs to be written first to
     // obtain its digest.
@@ -84,7 +84,8 @@ public CachedLayer writeLayer(LayerBuilder layerBuilder)
 
       CachedLayer cachedLayer = new CachedLayer(layerFile, compressedBlobDescriptor, diffId);
       LayerMetadata layerMetadata =
-          LayerMetadata.from(layerBuilder.getSourceFiles(), FileTime.from(Instant.now()));
+          LayerMetadata.from(
+              reproducibleLayerBuilder.getSourceFiles(), FileTime.from(Instant.now()));
       cache.addLayerToMetadata(cachedLayer, layerMetadata);
       return cachedLayer;
     }

jib-core/src/main/java/com/google/cloud/tools/jib/image/ImageReference.java

@@ -180,6 +180,10 @@ public String getTag() {
     return tag;
   }
 
+  public boolean usesDefaultTag() {
+    return DEFAULT_TAG.equals(tag);
+  }
+
   /** @return the image reference in Docker-readable format (inverse of {@link #parse}) */
   @Override
   public String toString() {

jib-core/src/main/java/com/google/cloud/tools/jib/image/LayerBuilder.java → jib-core/src/main/java/com/google/cloud/tools/jib/image/ReproducibleLayerBuilder.java

@@ -27,8 +27,12 @@
 import java.util.List;
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 
-/** Builds an {@link UnwrittenLayer} from files. */
-public class LayerBuilder {
+/**
+ * Builds a reproducible {@link UnwrittenLayer} from files. The reproducibility is implemented by
+ * strips out all non-reproducible elements (modification time, group ID, user ID, user name, and
+ * group name) from name-sorted tar archive entries.
+ */
+public class ReproducibleLayerBuilder {
 
   /**
    * The source files to build the layer from. Source files that are directories will have all
@@ -42,14 +46,9 @@ public class LayerBuilder {
   /** The Unix-style path of the file in the partial filesystem changeset. */
   private final String extractionPath;
 
-  /** Enable reproducible features when building the tar */
-  private final boolean enableReproducibleBuilds;
-
-  public LayerBuilder(
-      List<Path> sourceFiles, String extractionPath, boolean enableReproducibleBuilds) {
+  public ReproducibleLayerBuilder(List<Path> sourceFiles, String extractionPath) {
     this.sourceFiles = new ArrayList<>(sourceFiles);
     this.extractionPath = extractionPath;
-    this.enableReproducibleBuilds = enableReproducibleBuilds;
   }
 
   /** Builds and returns the layer. */
@@ -83,29 +82,21 @@ public UnwrittenLayer build() throws IOException {
       }
     }
 
-    if (enableReproducibleBuilds) {
-      makeListReproducible(filesystemEntries);
-    }
-
     // Adds all the files to a tar stream.
     TarStreamBuilder tarStreamBuilder = new TarStreamBuilder();
+    filesystemEntries.sort(Comparator.comparing(TarArchiveEntry::getName));
     for (TarArchiveEntry entry : filesystemEntries) {
-      tarStreamBuilder.addEntry(entry);
-    }
-
-    return new UnwrittenLayer(tarStreamBuilder.toBlob());
-  }
-
-  // Sort list and strip out all non-reproducible elements from tar archive entries.
-  private void makeListReproducible(List<TarArchiveEntry> entries) {
-    entries.sort(Comparator.comparing(TarArchiveEntry::getName));
-    for (TarArchiveEntry entry : entries) {
+      // Strips out all non-reproducible elements from tar archive entries.
       entry.setModTime(0);
       entry.setGroupId(0);
       entry.setUserId(0);
       entry.setUserName("");
       entry.setGroupName("");
+
+      tarStreamBuilder.addEntry(entry);
     }
+
+    return new UnwrittenLayer(tarStreamBuilder.toBlob());
   }
 
   public List<Path> getSourceFiles() {

jib-core/src/test/java/com/google/cloud/tools/jib/builder/BuildConfigurationTest.java

@@ -44,7 +44,6 @@ public void testBuilder() {
         Arrays.asList("credentialhelper", "anotherCredentialHelper");
     String expectedMainClass = "mainclass";
     RegistryCredentials expectedKnownRegistryCredentials = Mockito.mock(RegistryCredentials.class);
-    boolean expectedEnableReproducibleBuilds = false;
     List<String> expectedJvmFlags = Arrays.asList("some", "jvm", "flags");
     Map<String, String> expectedEnvironment = ImmutableMap.of("key", "value");
     Class<? extends BuildableManifestTemplate> expectedTargetFormat = OCIManifestTemplate.class;
@@ -59,7 +58,6 @@ public void testBuilder() {
                     expectedTargetServerUrl, expectedTargetImageName, expectedTargetTag))
             .setCredentialHelperNames(expectedCredentialHelperNames)
             .setKnownRegistryCredentials(expectedKnownRegistryCredentials)
-            .setEnableReproducibleBuilds(expectedEnableReproducibleBuilds)
             .setMainClass(expectedMainClass)
             .setJvmFlags(expectedJvmFlags)
             .setEnvironment(expectedEnvironment)
@@ -76,8 +74,6 @@ public void testBuilder() {
         expectedCredentialHelperNames, buildConfiguration.getCredentialHelperNames());
     Assert.assertEquals(
         expectedKnownRegistryCredentials, buildConfiguration.getKnownRegistryCredentials());
-    Assert.assertEquals(
-        expectedEnableReproducibleBuilds, buildConfiguration.getEnableReproducibleBuilds());
     Assert.assertEquals(expectedMainClass, buildConfiguration.getMainClass());
     Assert.assertEquals(expectedJvmFlags, buildConfiguration.getJvmFlags());
     Assert.assertEquals(expectedEnvironment, buildConfiguration.getEnvironment());
@@ -109,7 +105,6 @@ public void testBuilder_default() {
     Assert.assertEquals(Collections.emptyList(), buildConfiguration.getCredentialHelperNames());
     Assert.assertEquals(
         RegistryCredentials.none(), buildConfiguration.getKnownRegistryCredentials());
-    Assert.assertTrue(buildConfiguration.getEnableReproducibleBuilds());
     Assert.assertEquals(Collections.emptyList(), buildConfiguration.getJvmFlags());
     Assert.assertEquals(Collections.emptyMap(), buildConfiguration.getEnvironment());
     Assert.assertEquals(V22ManifestTemplate.class, buildConfiguration.getTargetFormat());

jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheWriterTest.java

@@ -21,8 +21,8 @@
 import com.google.cloud.tools.jib.blob.Blobs;
 import com.google.cloud.tools.jib.hash.CountingDigestOutputStream;
 import com.google.cloud.tools.jib.image.DescriptorDigest;
-import com.google.cloud.tools.jib.image.LayerBuilder;
 import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException;
+import com.google.cloud.tools.jib.image.ReproducibleLayerBuilder;
 import com.google.cloud.tools.jib.image.UnwrittenLayer;
 import com.google.common.io.CharStreams;
 import com.google.common.io.CountingOutputStream;
@@ -88,11 +88,12 @@ public void testWriteLayer_unwritten() throws IOException, LayerPropertyNotFound
     UnwrittenLayer unwrittenLayer = new UnwrittenLayer(Blobs.from(resourceBlob));
 
     List<Path> fakeSourceFiles = Collections.singletonList(Paths.get("some", "source", "file"));
-    LayerBuilder mockLayerBuilder = Mockito.mock(LayerBuilder.class);
-    Mockito.when(mockLayerBuilder.build()).thenReturn(unwrittenLayer);
-    Mockito.when(mockLayerBuilder.getSourceFiles()).thenReturn(fakeSourceFiles);
+    ReproducibleLayerBuilder mockReproducibleLayerBuilder =
+        Mockito.mock(ReproducibleLayerBuilder.class);
+    Mockito.when(mockReproducibleLayerBuilder.build()).thenReturn(unwrittenLayer);
+    Mockito.when(mockReproducibleLayerBuilder.getSourceFiles()).thenReturn(fakeSourceFiles);
 
-    CachedLayer cachedLayer = cacheWriter.writeLayer(mockLayerBuilder);
+    CachedLayer cachedLayer = cacheWriter.writeLayer(mockReproducibleLayerBuilder);
 
     CachedLayerWithMetadata layerInMetadata = testCache.getMetadata().getLayers().get(0);
     Assert.assertNotNull(layerInMetadata.getMetadata());