Separate out project dependencies, add integration test to verify (#1785

)

* Separate out project dependencies, add integration test to verify
* update changelog for gradle AND maven

jib-gradle-plugin/CHANGELOG.md

@@ -10,6 +10,8 @@ All notable changes to this project will be documented in this file.
 
 ### Changed
 
+- Dependencies are now split into three layers: dependencies, snapshots dependencies, project dependencies ([#1724](https://github.com/GoogleContainerTools/jib/pull/1724))
+
 ### Fixed
 
 ## 1.3.0

jib-gradle-plugin/src/integration-test/java/com/google/cloud/tools/jib/gradle/GradleLayerConfigurationIntegrationTest.java

@@ -0,0 +1,146 @@
+/*
+ * Copyright 2019 Google LLC.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.google.cloud.tools.jib.gradle;
+
+import com.google.common.collect.ImmutableList;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.zip.GZIPInputStream;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+public class GradleLayerConfigurationIntegrationTest {
+
+  @ClassRule
+  public static final TestProject multiTestProject = new TestProject("all-local-multi-service");
+
+  @Test
+  public void testGradleLayerConfiguration_multiModule() throws IOException {
+    multiTestProject.build(":complex-service:jibBuildTar");
+
+    Path jibTar = multiTestProject.getProjectRoot().resolve("complex-service/build/jib-image.tar");
+
+    List<List<String>> layers = getLayers(jibTar);
+
+    Assert.assertEquals(6, layers.size());
+
+    // the expected order is:
+    // no base image layers (scratch)
+    // extra-files (0)
+    // classes (1)
+    // resources (2)
+    // project dependencies (3)
+    // snapshot dependencies (4)
+    // dependencies (5)
+
+    Path complexServiceRoot = multiTestProject.getProjectRoot().resolve("complex-service");
+    // verify dependencies
+    List<String> dependencies = layers.get(5);
+    List<String> expectedDependencies =
+        ImmutableList.of("app/", "app/libs/", "app/libs/dependency-1.0.0.jar");
+    Assert.assertEquals(expectedDependencies, dependencies);
+
+    // verify snapshot dependencies
+    List<String> snapshotDependencies = layers.get(4);
+    List<String> expectedSnapshotDependencies =
+        ImmutableList.of("app/", "app/libs/", "app/libs/dependencyX-1.0.0-SNAPSHOT.jar");
+    Assert.assertEquals(expectedSnapshotDependencies, snapshotDependencies);
+
+    // verify project dependencies
+    List<String> projectDependencies = layers.get(3);
+    List<String> expectedProjectDependencies =
+        ImmutableList.of("app/", "app/libs/", "app/libs/lib.jar");
+    Assert.assertEquals(expectedProjectDependencies, projectDependencies);
+
+    // verify resources
+    List<String> resources = layers.get(2);
+    List<String> expectedResources =
+        ImmutableList.of(
+            "app/", "app/resources/", "app/resources/resource1.txt", "app/resources/resource2.txt");
+    Assert.assertEquals(expectedResources, resources);
+
+    // verify classes
+    List<String> classes = layers.get(1);
+    List<String> expectedClasses =
+        ImmutableList.of(
+            "app/",
+            "app/classes/",
+            "app/classes/com/",
+            "app/classes/com/test/",
+            "app/classes/com/test/HelloWorld.class");
+    Assert.assertEquals(expectedClasses, classes);
+
+    // verify extra files
+    List<String> extraFiles = layers.get(0);
+    List<String> expectedExtraFiles = ImmutableList.of("extra-file");
+    Assert.assertEquals(expectedExtraFiles, extraFiles);
+  }
+
+  @Test
+  public void testGradleLayerConfiguration_simpleModule() throws IOException {
+    multiTestProject.build(":simple-service:jibBuildTar");
+
+    Path jibTar = multiTestProject.getProjectRoot().resolve("simple-service/build/jib-image.tar");
+
+    List<List<String>> layers = getLayers(jibTar);
+
+    Assert.assertEquals(1, layers.size());
+
+    // the expected order is:
+    // no base image layers (scratch)
+    // classes (0)
+
+    // verify classes
+    List<String> classes = layers.get(0);
+    List<String> expectedClasses =
+        ImmutableList.of(
+            "app/",
+            "app/classes/",
+            "app/classes/com/",
+            "app/classes/com/test/",
+            "app/classes/com/test/HelloWorld.class");
+    Assert.assertEquals(expectedClasses, classes);
+  }
+
+  // returns all files in layers (*.tar.gz) in a image tar
+  private List<List<String>> getLayers(Path tar) throws IOException {
+    List<List<String>> layers = new ArrayList<>();
+
+    try (TarArchiveInputStream image = new TarArchiveInputStream(Files.newInputStream(tar))) {
+      TarArchiveEntry entry;
+      while ((entry = image.getNextTarEntry()) != null) {
+        if (entry.getName().endsWith(".tar.gz")) {
+          @SuppressWarnings("resource") // must not close sub-streams
+          TarArchiveInputStream layer = new TarArchiveInputStream(new GZIPInputStream(image));
+          TarArchiveEntry layerEntry;
+          List<String> layerFiles = new ArrayList<>();
+          while ((layerEntry = layer.getNextTarEntry()) != null) {
+            layerFiles.add(layerEntry.getName());
+          }
+          layers.add(0, layerFiles);
+        }
+      }
+    }
+    return layers;
+  }
+}

jib-gradle-plugin/src/main/java/com/google/cloud/tools/jib/gradle/GradleProjectProperties.java

@@ -48,8 +48,11 @@
 import org.gradle.api.GradleException;
 import org.gradle.api.JavaVersion;
 import org.gradle.api.Project;
+import org.gradle.api.artifacts.ResolvedArtifact;
+import org.gradle.api.artifacts.component.ProjectComponentIdentifier;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.logging.Logger;
+import org.gradle.api.plugins.JavaPlugin;
 import org.gradle.api.plugins.JavaPluginConvention;
 import org.gradle.api.plugins.WarPlugin;
 import org.gradle.api.tasks.SourceSet;
@@ -149,21 +152,43 @@ public JibContainerBuilder createContainerBuilder(
       Path resourcesOutputDirectory = mainSourceSet.getOutput().getResourcesDir().toPath();
       FileCollection allFiles = mainSourceSet.getRuntimeClasspath().filter(File::exists);
 
-      FileCollection allDependencyFiles =
+      FileCollection projectDependencies =
+          project.files(
+              project
+                  .getConfigurations()
+                  .getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME)
+                  .getResolvedConfiguration()
+                  .getResolvedArtifacts()
+                  .stream()
+                  .filter(
+                      artifact ->
+                          artifact.getId().getComponentIdentifier()
+                              instanceof ProjectComponentIdentifier)
+                  .map(ResolvedArtifact::getFile)
+                  .collect(Collectors.toList()));
+
+      FileCollection nonProjectDependencies =
           allFiles
               .minus(classesOutputDirectories)
+              .minus(projectDependencies)
               .filter(file -> !file.toPath().equals(resourcesOutputDirectory));
 
-      FileCollection snapshotDependencyFiles =
-          allDependencyFiles.filter(file -> file.getName().contains("SNAPSHOT"));
-      FileCollection dependencyFiles = allDependencyFiles.minus(snapshotDependencyFiles);
+      FileCollection snapshotDependencies =
+          nonProjectDependencies.filter(file -> file.getName().contains("SNAPSHOT"));
+      FileCollection dependencies = nonProjectDependencies.minus(snapshotDependencies);
 
       // Adds dependency files
       javaContainerBuilder
           .addDependencies(
-              dependencyFiles.getFiles().stream().map(File::toPath).collect(Collectors.toList()))
+              dependencies.getFiles().stream().map(File::toPath).collect(Collectors.toList()))
           .addSnapshotDependencies(
-              snapshotDependencyFiles
+              snapshotDependencies
+                  .getFiles()
+                  .stream()
+                  .map(File::toPath)
+                  .collect(Collectors.toList()))
+          .addProjectDependencies(
+              projectDependencies
                   .getFiles()
                   .stream()
                   .map(File::toPath)

jib-gradle-plugin/src/test/java/com/google/cloud/tools/jib/gradle/GradleProjectPropertiesTest.java

@@ -51,6 +51,8 @@
 import org.gradle.StartParameter;
 import org.gradle.api.JavaVersion;
 import org.gradle.api.Project;
+import org.gradle.api.artifacts.ConfigurationContainer;
+import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.internal.file.AbstractFileCollection;
 import org.gradle.api.internal.file.FileResolver;
@@ -60,6 +62,7 @@
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.configuration.ConsoleOutput;
 import org.gradle.api.plugins.Convention;
+import org.gradle.api.plugins.JavaPlugin;
 import org.gradle.api.plugins.JavaPluginConvention;
 import org.gradle.api.plugins.PluginContainer;
 import org.gradle.api.plugins.WarPlugin;
@@ -198,6 +201,21 @@ public void setup() throws URISyntaxException, IOException {
     Mockito.when(mockGradle.getStartParameter()).thenReturn(mockStartParameter);
     Mockito.when(mockStartParameter.getConsoleOutput()).thenReturn(ConsoleOutput.Auto);
 
+    // mocking to complete ignore project dependency resolution
+    Mockito.when(mockProject.getConfigurations())
+        .thenReturn(Mockito.mock(ConfigurationContainer.class, Mockito.RETURNS_DEEP_STUBS));
+    Mockito.when(
+            mockProject
+                .getConfigurations()
+                .getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME)
+                .getResolvedConfiguration()
+                .getResolvedArtifacts())
+        .thenReturn(ImmutableSet.of());
+    ConfigurableFileCollection emptyFileCollection = Mockito.mock(ConfigurableFileCollection.class);
+    Mockito.when(emptyFileCollection.getFiles()).thenReturn(ImmutableSet.of());
+    Mockito.when(mockProject.files(ImmutableList.of())).thenReturn(emptyFileCollection);
+    // done mocking project dependency resolution
+
     Set<Path> classesFiles = ImmutableSet.of(getResource("gradle/application/classes"));
     FileCollection classesFileCollection = new TestFileCollection(classesFiles);
     Path resourcesOutputDir = getResource("gradle/application/resources");

jib-gradle-plugin/src/test/resources/gradle/projects/all-local-multi-service/build.gradle

@@ -0,0 +1 @@
+// this file doesn't do anything

jib-gradle-plugin/src/test/resources/gradle/projects/all-local-multi-service/complex-service/build.gradle

@@ -0,0 +1,36 @@
+plugins {
+  id 'java'
+  id 'com.google.cloud.tools.jib'
+}
+
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
+
+repositories {
+  flatDir {
+    dirs "libs"
+  }
+}
+
+jib {
+  from {
+    image = "scratch"
+  }
+  extraDirectories {
+    paths = file('src/main/other-jib')
+  }
+}
+
+sourceSets {
+  main {
+    resources {
+      srcDirs "src/main/extra-resources-1", "src/main/extra-resources-2"
+    }
+  }
+}
+
+dependencies {
+  compile project(':lib')
+  compile name: "dependency-1.0.0"
+  compile name: "dependencyX-1.0.0-SNAPSHOT"
+}

jib-gradle-plugin/src/test/resources/gradle/projects/all-local-multi-service/complex-service/src/main/java/com/test/HelloWorld.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2018 Google LLC.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.test;
+
+public class HelloWorld {
+  public static void main(String[] args) {
+    System.out.println("Hello world");
+  }
+}

jib-gradle-plugin/src/test/resources/gradle/projects/all-local-multi-service/lib/build.gradle

@@ -0,0 +1,10 @@
+plugins {
+  id 'java'
+}
+
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
+
+repositories {
+  mavenCentral()
+}

jib-gradle-plugin/src/test/resources/gradle/projects/all-local-multi-service/lib/src/main/java/com/lib/Lib.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2018 Google LLC.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.lib;
+
+/** Shared Code! */
+public class Lib {
+
+  public String getThing() {
+    return "thing";
+  }
+}

jib-gradle-plugin/src/test/resources/gradle/projects/all-local-multi-service/lib/src/main/resources/hi.txt

@@ -0,0 +1 @@
+hi

jib-gradle-plugin/src/test/resources/gradle/projects/all-local-multi-service/settings.gradle

@@ -0,0 +1,3 @@
+include ':simple-service'
+include ':complex-service'
+include ':lib'

jib-gradle-plugin/src/test/resources/gradle/projects/all-local-multi-service/simple-service/build.gradle

@@ -0,0 +1,18 @@
+plugins {
+  id 'java'
+  id 'com.google.cloud.tools.jib'
+}
+
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
+
+repositories {
+  mavenCentral()
+}
+
+jib {
+  from {
+    image = "scratch"
+  }
+    // only use buildTar
+}