#7472: ensure deterministic classpath order based on maven 'dependenc…

…y:build-classpath' goal

kernel/base/src/main/java/com/twosigma/beakerx/kernel/magic/command/MavenJarResolver.java

@@ -15,32 +15,33 @@
  */
 package com.twosigma.beakerx.kernel.magic.command;
 
-import com.twosigma.beakerx.util.Preconditions;
 import com.twosigma.beakerx.kernel.commands.MavenInvocationSilentOutputHandler;
 import com.twosigma.beakerx.kernel.commands.MavenJarResolverSilentLogger;
 import com.twosigma.beakerx.kernel.magic.command.functionality.MvnLoggerWidget;
+import com.twosigma.beakerx.util.Preconditions;
 import org.apache.commons.io.FileUtils;
-import org.apache.maven.shared.invoker.DefaultInvocationRequest;
-import org.apache.maven.shared.invoker.DefaultInvoker;
-import org.apache.maven.shared.invoker.InvocationRequest;
-import org.apache.maven.shared.invoker.InvocationResult;
-import org.apache.maven.shared.invoker.Invoker;
+import org.apache.commons.io.IOUtils;
+import org.apache.maven.shared.invoker.*;
 
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.UUID;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static java.util.Collections.singletonList;
 
 public class MavenJarResolver {
 
   public static final String MVN_DIR = File.separator + "mvnJars";
   public static final String POM_XML = "PomTemplateMagicCommand.xml";
-  public static final List<String> GOALS = Collections.singletonList("validate");
+  public static final String GOAL = "validate";
+  public static final String MAVEN_BUILT_CLASSPATH_FILE_NAME = "mavenclasspathfilename.txt";
 
   private final String pathToMavenRepo;
   private ResolverParams commandParams;
@@ -55,14 +56,14 @@ public MavenJarResolver(final ResolverParams commandParams,
   }
 
   public AddMvnCommandResult retrieve(Dependency dependency, MvnLoggerWidget progress) {
-    List<Dependency> dependencies = Arrays.asList(dependency);
+    List<Dependency> dependencies = singletonList(dependency);
     return retrieve(dependencies, progress);
   }
 
   public AddMvnCommandResult retrieve(List<Dependency> dependencies, MvnLoggerWidget progress) {
     File finalPom = null;
     try {
-      String pomAsString = pomFactory.createPom(pathToMavenRepo, dependencies, commandParams.getRepos());
+      String pomAsString = pomFactory.createPom(new PomFactory.Params(pathToMavenRepo, dependencies, commandParams.getRepos(), GOAL, MAVEN_BUILT_CLASSPATH_FILE_NAME));
       finalPom = saveToFile(commandParams.getPathToNotebookJars(), pomAsString);
       InvocationRequest request = createInvocationRequest();
       request.setOffline(commandParams.getOffline());
@@ -134,12 +135,42 @@ private AddMvnCommandResult getResult(InvocationResult invocationResult, List<De
       return AddMvnCommandResult.error(errorMsgBuilder.toString());
     }
 
-    return AddMvnCommandResult.SUCCESS;
+    return AddMvnCommandResult.success(transformFromMavenRepoToKernelRepo(mavenBuildClasspath(), jarsFromRepo()));
+  }
+
+  private List<String> transformFromMavenRepoToKernelRepo(List<String> jarNamesFromBuildClasspath, Map<String, Path> jarNames) {
+    List<String> result = new ArrayList<>();
+    jarNamesFromBuildClasspath.forEach(jarName -> {
+      result.add(jarNames.get(jarName).toAbsolutePath().toString());
+    });
+    return result;
+  }
+
+  private Map<String, Path> jarsFromRepo() {
+    try {
+      List<Path> collect = Files.list(Paths.get(pathToMavenRepo)).collect(Collectors.toList());
+      return collect.stream().collect(Collectors.toMap(x -> x.getFileName().toString(), x -> x));
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private List<String> mavenBuildClasspath() {
+    String jarPathsAsString = null;
+    try {
+      String absolutePath = new File(pathToMavenRepo).getAbsolutePath() + File.separator + MAVEN_BUILT_CLASSPATH_FILE_NAME;
+      InputStream isPaths = Files.newInputStream(Paths.get(absolutePath));
+      jarPathsAsString = IOUtils.toString(isPaths, StandardCharsets.UTF_8);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    Stream<String> stream = Arrays.stream(jarPathsAsString.split(File.pathSeparator));
+    return stream.map(x -> Paths.get(x).getFileName().toString()).collect(Collectors.toList());
   }
 
   private InvocationRequest createInvocationRequest() {
     InvocationRequest request = new DefaultInvocationRequest();
-    request.setGoals(GOALS);
+    request.setGoals(singletonList(GOAL));
     return request;
   }
 
@@ -207,14 +238,14 @@ public Optional<String> getClassifier() {
 
   public static class AddMvnCommandResult {
 
-    public static final AddMvnCommandResult SUCCESS = new AddMvnCommandResult(true, "");
-
     private boolean jarRetrieved;
     private String errorMessage;
+    private List<String> addedJarsPaths;
 
-    private AddMvnCommandResult(boolean retrieved, String errorMessage) {
+    private AddMvnCommandResult(boolean retrieved, String errorMessage, List<String> addedJarsPaths) {
       this.jarRetrieved = retrieved;
       this.errorMessage = errorMessage;
+      this.addedJarsPaths = addedJarsPaths;
     }
 
     public boolean isJarRetrieved() {
@@ -225,12 +256,16 @@ public String getErrorMessage() {
       return errorMessage;
     }
 
-    public static AddMvnCommandResult success() {
-      return SUCCESS;
+    public static AddMvnCommandResult success(List<String> addedJarsPaths) {
+      return new AddMvnCommandResult(true, "", addedJarsPaths);
     }
 
     public static AddMvnCommandResult error(String msg) {
-      return new AddMvnCommandResult(false, msg);
+      return new AddMvnCommandResult(false, msg, new ArrayList<>());
+    }
+
+    public List<String> getAddedJarPaths() {
+      return addedJarsPaths;
     }
   }
 
@@ -247,12 +282,6 @@ public ResolverParams(String pathToCache, String pathToNotebookJars, boolean off
       this.offline = offline;
     }
 
-    public ResolverParams(String pathToCache, String pathToNotebookJars, boolean offline,
-                          Map<String, String> repos) {
-      this(pathToCache, pathToNotebookJars, offline);
-      this.repos = repos;
-    }
-
     public ResolverParams(String pathToCache, String pathToNotebookJars) {
       this(pathToCache, pathToNotebookJars, false);
     }

kernel/base/src/main/java/com/twosigma/beakerx/kernel/magic/command/PomFactory.java

@@ -29,15 +29,28 @@
 
 public class PomFactory {
 
-  public String createPom(String pathToMavenRepo, List<Dependency> dependencies, Map<String, String> repos) throws IOException {
+  public String createPom(Params params) throws IOException {
     InputStream pom = getClass().getClassLoader().getResourceAsStream(MavenJarResolver.POM_XML);
     String pomAsString = IOUtils.toString(pom, StandardCharsets.UTF_8);
-    pomAsString = configureOutputDir(pathToMavenRepo, pomAsString);
-    pomAsString = configureDependencies(dependencies, pomAsString);
-    pomAsString = configureRepos(repos, pomAsString);
+    pomAsString = configureOutputDir(params.pathToMavenRepo, pomAsString);
+    pomAsString = configureDependencies(params.dependencies, pomAsString);
+    pomAsString = configureRepos(params.repos, pomAsString);
+    pomAsString = configureGoal(params.goal, pomAsString);
+    pomAsString = configureBuildClasspathPlugin(params.pathToMavenRepo, params.mavenBuiltClasspathFileName, pomAsString);
     return pomAsString;
   }
 
+  private String configureBuildClasspathPlugin(String pathToMavenRepo, String mavenBuiltClasspathFileName, String pomAsString) {
+    String absolutePath = new File(pathToMavenRepo).getAbsolutePath();
+    return pomAsString.replaceAll(
+            "<outputFile>mavenBuiltClasspathFile</outputFile>",
+            "<outputFile>" + absolutePath + File.separator +mavenBuiltClasspathFileName + "</outputFile>");
+  }
+
+  private String configureGoal(String goal, String pomAsString) {
+    return pomAsString.replaceAll("<phase>goal</phase>", "<phase>" + goal + "</phase>");
+  }
+
   private String configureDependencies(List<Dependency> dependencies, String pomAsString) {
     for (Dependency dependency : dependencies) {
       pomAsString = configureDependency(dependency, pomAsString);
@@ -87,4 +100,20 @@ private String configureRepos(String pomAsString, String name, String url) {
 
     return pomAsString.replace("</repositories>", String.format(repoPattern, name, url) + "</repositories>");
   }
+
+  static class Params {
+    String pathToMavenRepo;
+    List<Dependency> dependencies;
+    Map<String, String> repos;
+    String goal;
+    private String mavenBuiltClasspathFileName;
+
+    public Params(String pathToMavenRepo, List<Dependency> dependencies, Map<String, String> repos, String goal, String mavenBuiltClasspathFileName) {
+      this.pathToMavenRepo = pathToMavenRepo;
+      this.dependencies = dependencies;
+      this.repos = repos;
+      this.goal = goal;
+      this.mavenBuiltClasspathFileName = mavenBuiltClasspathFileName;
+    }
+  }
 }

kernel/base/src/main/java/com/twosigma/beakerx/kernel/magic/command/functionality/ClassPathAddMvnCellMagicCommand.java

@@ -90,7 +90,7 @@ public MagicCommandOutcomeItem execute(String command, String commandCodeBlock,
     MavenJarResolver.AddMvnCommandResult result = mavenJarResolver.retrieve(dependencies, mvnLoggerWidget);
 
     if (result.isJarRetrieved()) {
-      return handleAddedJars(mavenJarResolver.getPathToMavenRepo() + "/*");
+      return handleAddedJars(result.getAddedJarPaths());
     }
     return new MagicCommandOutput(MagicCommandOutput.Status.ERROR, result.getErrorMessage());
   }

kernel/base/src/main/java/com/twosigma/beakerx/kernel/magic/command/functionality/ClasspathAddMvnMagicCommand.java

@@ -82,7 +82,7 @@ public MagicCommandOutcomeItem execute(String command, Message message) {
     MvnLoggerWidget progress = new MvnLoggerWidget(message);
     AddMvnCommandResult result = retrieve(getDependency(split), classpathAddMvnCommand, progress);
     if (result.isJarRetrieved()) {
-      return handleAddedJars(classpathAddMvnCommand.getPathToMavenRepo() + "/*");
+      return handleAddedJars(result.getAddedJarPaths());
     }
     return new MagicCommandOutput(MagicCommandOutput.Status.ERROR, result.getErrorMessage());
   }

kernel/base/src/main/java/com/twosigma/beakerx/kernel/magic/command/functionality/ClasspathMagicCommand.java

@@ -21,6 +21,7 @@
 import com.twosigma.beakerx.kernel.magic.command.outcome.MagicCommandOutcomeItem;
 import com.twosigma.beakerx.kernel.magic.command.outcome.MagicCommandOutput;
 import com.twosigma.beakerx.kernel.magic.command.outcome.MagicCommandOutputFoldout;
+import org.jetbrains.annotations.NotNull;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -46,7 +47,8 @@ public ClasspathMagicCommand(KernelFunctionality kernel) {
   }
 
   public Collection<String> addJars(Collection<String> path) {
-    return path.stream().map(this::addJars).flatMap(Collection::stream).collect(Collectors.toList());
+    List<PathToJar> collect = path.stream().map(PathToJar::new).collect(Collectors.toList());
+    return addJarsToClasspath(collect);
   }
 
   public Collection<String> addJars(String path) {
@@ -87,12 +89,18 @@ private Collection<String> handlePath(String path) {
   }
 
   private List<String> handleWildCards(String path) {
-    List<String> addedJarsName = new LinkedList<>();
     Map<Path, String> paths = getPaths(path);
     List<PathToJar> pathsToJars = paths.keySet().stream()
             .map(currentPath -> new PathToJar(currentPath.toString()))
             .collect(Collectors.toList());
 
+    List<String> addedJarsName = addJarsToClasspath(pathsToJars);
+    return addedJarsName;
+  }
+
+  @NotNull
+  private List<String> addJarsToClasspath(List<PathToJar> pathsToJars) {
+    List<String> addedJarsName = new LinkedList<>();
     List<Path> addedPaths = kernel.addJarsToClasspath(pathsToJars);
     addedJarsName.addAll(addedPaths.stream().map(x -> x.getFileName().toString()).collect(Collectors.toList()));
     return addedJarsName;

kernel/base/src/main/resources/PomTemplateMagicCommand.xml

@@ -21,7 +21,7 @@
                 <executions>
                     <execution>
                         <id>copy-dependencies</id>
-                        <phase>validate</phase>
+                        <phase>goal</phase>
                         <goals>
                             <goal>copy-dependencies</goal>
                         </goals>
@@ -32,6 +32,16 @@
                             <overWriteIfNewer>true</overWriteIfNewer>
                         </configuration>
                     </execution>
+                    <execution>
+                        <id>build-classpath</id>
+                        <phase>goal</phase>
+                        <goals>
+                            <goal>build-classpath</goal>
+                        </goals>
+                        <configuration>
+                            <outputFile>mavenBuiltClasspathFile</outputFile>
+                        </configuration>
+                    </execution>
                 </executions>
             </plugin>
         </plugins>

kernel/base/src/test/java/com/twosigma/beakerx/kernel/magic/command/ClasspathAddMvnDepsMagicCommandTest.java

@@ -47,9 +47,8 @@
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
-import static com.twosigma.beakerx.kernel.magic.command.functionality.ClasspathAddMvnMagicCommand.ADD_MVN_FORMAT_ERROR_MESSAGE;
-import static com.twosigma.beakerx.kernel.magic.command.functionality.ClasspathAddMvnMagicCommand.CLASSPATH_ADD_MVN;
-import static com.twosigma.beakerx.kernel.magic.command.functionality.ClasspathAddMvnMagicCommand.DEFAULT_MAVEN_REPOS;
+import static com.twosigma.beakerx.kernel.magic.command.MavenJarResolver.MAVEN_BUILT_CLASSPATH_FILE_NAME;
+import static com.twosigma.beakerx.kernel.magic.command.functionality.ClasspathAddMvnMagicCommand.*;
 import static com.twosigma.beakerx.kernel.magic.command.functionality.ClasspathResetMagicCommand.CLASSPATH_RESET;
 import static java.util.Collections.singletonList;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -161,6 +160,7 @@ private void handleClasspathAddMvnDep(String allCode, String expected) throws Ex
             file.getFileName().toFile().getName().contains("slf4j"))).findFirst();
     assertThat(dep).isPresent();
     assertThat(kernel.getClasspath().get(0)).contains(mvnDir);
+    assertThat(Files.exists(Paths.get(kernel.mavenResolverParam.getPathToNotebookJars() + File.separator + MAVEN_BUILT_CLASSPATH_FILE_NAME))).isTrue();
     dep.ifPresent(path -> {
       try {
         FileUtils.forceDelete(path.toFile());

kernel/base/src/test/java/com/twosigma/beakerx/kernel/magic/command/PomFactoryTest.java

@@ -76,7 +76,7 @@ public void createPomWithRepos() throws Exception {
     repos.put("repository.spring.snapshot", "http://repo.spring.io/snapshot");
     Dependency dependency = Dependency.create(asList("", "", ""));
     //when
-    String pomAsString = pomFactory.createPom("/", asList(dependency), repos);
+    String pomAsString = pomFactory.createPom(new PomFactory.Params("/", asList(dependency), repos, MavenJarResolver.GOAL, MavenJarResolver.MAVEN_BUILT_CLASSPATH_FILE_NAME));
     //then
     assertThat(removeWhitespaces(pomAsString)).contains(removeWhitespaces(EXPECTED_RESULT_BLOCK));
   }
@@ -86,7 +86,7 @@ public void createPomWithMultipleDependencies() throws Exception {
     Map<String, String> repos = Maps.newHashMap();
     Dependency dependency1 = Dependency.create(asList("group", "artifact", "1.1.1"));
     Dependency dependency2 = Dependency.create(asList("other-group", "other-artifact", "1.1.1"));
-    String pomAsString = pomFactory.createPom("/", asList(dependency1, dependency2), repos);
+    String pomAsString = pomFactory.createPom(new PomFactory.Params("/", asList(dependency1, dependency2), repos, MavenJarResolver.GOAL, MavenJarResolver.MAVEN_BUILT_CLASSPATH_FILE_NAME));
     assertThat(removeWhitespaces(pomAsString)).contains(removeWhitespaces(EXPECTED_MULTIPLE_DEP_POM));
   }
 
@@ -95,7 +95,7 @@ public void createPomWithType() throws Exception {
     //given
     Dependency dependency = Dependency.create(asList("group", "art", "ver", "pom"));
     //when
-    String pomAsString = pomFactory.createPom("/", asList(dependency), Maps.newHashMap());
+    String pomAsString = pomFactory.createPom(new PomFactory.Params("/", asList(dependency), Maps.newHashMap(), MavenJarResolver.GOAL, MavenJarResolver.MAVEN_BUILT_CLASSPATH_FILE_NAME));
     //then
     assertThat(removeWhitespaces(pomAsString)).contains("<type>pom</type>");
     assertThat(removeWhitespaces(pomAsString)).doesNotContain("<classifier");
@@ -106,7 +106,7 @@ public void createPomWithClassifier() throws Exception {
     //given
     Dependency dependency = Dependency.create(asList("group", "art", "ver", "pom", "Classifier"));
     //when
-    String pomAsString = pomFactory.createPom("/", asList(dependency), Maps.newHashMap());
+    String pomAsString = pomFactory.createPom(new PomFactory.Params("/", asList(dependency), Maps.newHashMap(), MavenJarResolver.GOAL, MavenJarResolver.MAVEN_BUILT_CLASSPATH_FILE_NAME));
     //then
     assertThat(removeWhitespaces(pomAsString)).contains("<classifier>Classifier</classifier>");
   }

kernel/base/src/test/resources/PomTemplateMagicCommand.xml

@@ -1,4 +1,3 @@
-<?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0"
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
@@ -26,7 +25,7 @@
                 <executions>
                     <execution>
                         <id>copy-dependencies</id>
-                        <phase>validate</phase>
+                        <phase>goal</phase>
                         <goals>
                             <goal>copy-dependencies</goal>
                         </goals>
@@ -37,6 +36,16 @@
                             <overWriteIfNewer>true</overWriteIfNewer>
                         </configuration>
                     </execution>
+                    <execution>
+                        <id>build-classpath</id>
+                        <phase>goal</phase>
+                        <goals>
+                            <goal>build-classpath</goal>
+                        </goals>
+                        <configuration>
+                            <outputFile>mavenBuiltClasspathFile</outputFile>
+                        </configuration>
+                    </execution>
                 </executions>
             </plugin>
         </plugins>