diff --git a/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/extractor/MetadataCollector.java b/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/extractor/MetadataCollector.java index 19d153bf..8ad8f185 100644 --- a/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/extractor/MetadataCollector.java +++ b/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/extractor/MetadataCollector.java @@ -6,10 +6,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.LinkedList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; import org.openrewrite.ExecutionContext; @@ -59,8 +60,8 @@ public String getDescription() { public static class MetadataAccumulator { private final List commonFiles = new ArrayList<>(); private final List otherFiles = new ArrayList<>(); - private final List flags = new LinkedList<>(); - private final List jdkVersions = new ArrayList<>(); + private final Set flags = new HashSet<>(); + private final Set jdkVersions = new HashSet<>(); public List getCommonFiles() { return commonFiles; @@ -70,7 +71,7 @@ public List getOtherFiles() { return otherFiles; } - public List getJdkVersions() { + public Set getJdkVersions() { return jdkVersions; } @@ -86,7 +87,7 @@ public void addJdk(JDK jdk) { jdkVersions.add(jdk); } - public List getFlags() { + public Set getFlags() { return flags; } diff --git a/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/extractor/PluginMetadata.java b/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/extractor/PluginMetadata.java index 5991e346..4092583a 100644 --- a/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/extractor/PluginMetadata.java +++ b/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/extractor/PluginMetadata.java @@ -4,10 +4,12 @@ import io.jenkins.tools.pluginmodernizer.core.model.CacheEntry; import io.jenkins.tools.pluginmodernizer.core.model.JDK; import io.jenkins.tools.pluginmodernizer.core.model.Plugin; +import io.jenkins.tools.pluginmodernizer.core.model.PreconditionError; import java.io.Serializable; import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.Set; /** * Metadata of a plugin extracted from its POM file or code @@ -22,7 +24,12 @@ public class PluginMetadata extends CacheEntry implements Serial /** * List of flags present in the plugin */ - private List flags; + private Set flags; + + /** + * List of errors present in the plugin + */ + private Set errors; /** * List of well known files present in the plugin @@ -37,7 +44,7 @@ public class PluginMetadata extends CacheEntry implements Serial /** * JDK versions supported by the plugin */ - private List jdkVersions; + private Set jdkVersions; /** * Jenkins version required by the plugin @@ -91,11 +98,11 @@ public void setPluginName(String pluginName) { this.pluginName = pluginName; } - public List getFlags() { + public Set getFlags() { return flags; } - public void setFlags(List flags) { + public void setFlags(Set flags) { this.flags = flags; } @@ -103,6 +110,14 @@ public boolean hasFlag(MetadataFlag flag) { return flags.contains(flag); } + public Set getErrors() { + return errors; + } + + public void setErrors(Set errors) { + this.errors = errors; + } + public List getCommonFiles() { return commonFiles; } @@ -119,11 +134,11 @@ public void setOtherFiles(List otherFiles) { this.otherFiles = otherFiles; } - public List getJdks() { + public Set getJdks() { return jdkVersions; } - public void setJdks(List jdkVersions) { + public void setJdks(Set jdkVersions) { this.jdkVersions = jdkVersions; } diff --git a/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/impl/PluginModernizer.java b/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/impl/PluginModernizer.java index 5d681c8c..7c660b2c 100644 --- a/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/impl/PluginModernizer.java +++ b/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/impl/PluginModernizer.java @@ -12,6 +12,8 @@ import io.jenkins.tools.pluginmodernizer.core.utils.UpdateCenterUtils; import java.nio.file.Path; import java.util.List; +import java.util.stream.Collectors; +import org.openrewrite.Recipe; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,7 +55,8 @@ public void start() { // Debug config LOG.debug("Plugins: {}", config.getPlugins()); - LOG.debug("Recipes: {}", config.getRecipes()); + LOG.debug( + "Recipes: {}", config.getRecipes().stream().map(Recipe::getName).collect(Collectors.joining(", "))); LOG.debug("GitHub owner: {}", config.getGithubOwner()); LOG.debug("Update Center Url: {}", config.getJenkinsUpdateCenter()); LOG.debug("Cache Path: {}", config.getCachePath()); @@ -107,22 +110,18 @@ private void process(Plugin plugin) { // Compile only if we are able to find metadata // For the moment it's local cache only but later will fetch on remote storage if (!config.isFetchMetadataOnly()) { - if (plugin.getMetadata() != null) { + if (plugin.getMetadata() != null && !plugin.hasPreconditionErrors()) { JDK jdk = compilePlugin(plugin); LOG.info("Plugin {} compiled successfully with JDK {}", plugin.getName(), jdk.getMajor()); } else { - LOG.info("No metadata found for plugin {}. Skipping initial compilation.", plugin.getName()); + LOG.info( + "No metadata or precondition errors found for plugin {}. Skipping initial compilation.", + plugin.getName()); } } plugin.checkoutBranch(ghService); - // Ensure minimum baseline of JDK 8. - // For the moment some plugin cannot be refreshed due to some condition - // (Non HTTPS URL, java 7 build). For those plugin we will fail until we find a solution - LOG.info("Checking if plugin {} can be modernized", plugin.getName()); - plugin.ensureMinimalBuild(mavenInvoker); - // Minimum JDK to run openrewrite plugin.withJDK(JDK.JAVA_17); @@ -131,8 +130,7 @@ private void process(Plugin plugin) { plugin.collectMetadata(mavenInvoker); - CacheManager pluginCacheManager = new CacheManager(Path.of(Settings.TEST_PLUGINS_DIRECTORY) - .resolve(plugin.getLocalRepository().resolve("target"))); + CacheManager pluginCacheManager = plugin.buildPluginTargetDirectoryCacheManager(); plugin.setMetadata(pluginCacheManager.move( cacheManager, Path.of(plugin.getName()), @@ -146,6 +144,15 @@ private void process(Plugin plugin) { LOG.debug("Metadata already computed for plugin {}. Using cached metadata.", plugin.getName()); } + // Abort here if we have errors + if (plugin.hasErrors() || plugin.hasPreconditionErrors()) { + plugin.addPreconditionErrors(plugin.getMetadata()); + LOG.info( + "Skipping plugin {} due to metadata/precondition errors. Check logs for more details.", + plugin.getName()); + return; + } + // Run OpenRewrite plugin.runOpenRewrite(mavenInvoker); if (plugin.hasErrors()) { diff --git a/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/model/CacheEntry.java b/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/model/CacheEntry.java index 909d9467..d448012d 100644 --- a/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/model/CacheEntry.java +++ b/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/model/CacheEntry.java @@ -115,7 +115,8 @@ public final T move(CacheManager newCacheManager, Path newPath, String newKey) { */ public final void save() { LOG.debug( - "Saving object to {}", cacheManager.getLocation().resolve(path).resolve(key)); + "Saving object to {}", + cacheManager.getLocation().resolve(path).resolve(key).toAbsolutePath()); cacheManager.put(this); } diff --git a/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/model/JDK.java b/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/model/JDK.java index 60204c4a..8c834fba 100644 --- a/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/model/JDK.java +++ b/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/model/JDK.java @@ -8,6 +8,7 @@ import java.util.Arrays; import java.util.Comparator; import java.util.List; +import java.util.Set; import org.apache.maven.artifact.versioning.ComparableVersion; /** @@ -213,7 +214,7 @@ public static JDK min() { * @param jdks List of JDKS. Can be null or empty * @return The minimum JDK. If the list is empty, return the minimum JDK available */ - public static JDK min(List jdks) { + public static JDK min(Set jdks) { if (jdks == null || jdks.isEmpty()) { return JDK.min(); } diff --git a/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/model/Plugin.java b/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/model/Plugin.java index 4d03e188..09c6de68 100644 --- a/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/model/Plugin.java +++ b/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/model/Plugin.java @@ -4,20 +4,22 @@ import io.jenkins.tools.pluginmodernizer.core.config.Settings; import io.jenkins.tools.pluginmodernizer.core.extractor.PluginMetadata; import io.jenkins.tools.pluginmodernizer.core.github.GHService; +import io.jenkins.tools.pluginmodernizer.core.impl.CacheManager; import io.jenkins.tools.pluginmodernizer.core.impl.MavenInvoker; import java.io.File; import java.io.IOException; import java.net.URI; import java.nio.file.Path; +import java.util.Arrays; import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathFactory; import org.apache.commons.io.FileUtils; import org.kohsuke.github.GHRepository; @@ -221,6 +223,26 @@ public boolean hasErrors() { return !errors.isEmpty(); } + /** + * Return if the plugin has any precondition errors + * @return True if the plugin has precondition errors + */ + public boolean hasPreconditionErrors() { + return metadata != null + && metadata.getErrors() != null + && !metadata.getErrors().isEmpty(); + } + + /** + * Add precondition errors to the plugin errors + */ + public void addPreconditionErrors(PluginMetadata metadata) { + if (metadata == null) { + return; + } + metadata.getErrors().forEach(error -> addError(error.getError())); + } + /** * Get the errors of the plugin * @return List of errors @@ -424,95 +446,32 @@ public void verify(MavenInvoker maven) { * @param maven The maven invoker instance */ public void collectMetadata(MavenInvoker maven) { - maven.collectMetadata(this); - } - /** - * Ensure a minimal build can be performed on the plugin. - * Some plugin are very outdated they cannot compile anymore due to non-https URL - * and missing relative path on the parent pom. This methods ensure that the plugin - * is setup correctly before attempting to compile it. - * @param maven The maven invoker instance - */ - public void ensureMinimalBuild(MavenInvoker maven) { + XPathFactory xPathFactory = XPathFactory.newInstance(); + XPath xpath = xPathFactory.newXPath(); // Static parse of the pom file and check for pattern preventing minimal build Path pom = getLocalRepository().resolve("pom.xml"); + if (!getLocalRepository().resolve("target").toFile().mkdir()) { + LOG.debug("Failed to create target directory for plugin {}", name); + } Document document = staticPomParse(pom); - // Perform checks - checkStaticPom(document); - - LOG.info("Ensuring minimal plugin {} build ... Please be patient", name); - maven.ensureMinimalBuild(this); - } + // Collect precondition errors + PluginMetadata pluginMetadata = new PluginMetadata(); + pluginMetadata.setCacheManager(buildPluginTargetDirectoryCacheManager()); + pluginMetadata.setErrors(Arrays.stream(PreconditionError.values()) + .filter(error -> error.isApplicable(document, xpath)) + .collect(Collectors.toSet())); - /** - * Static parse of the pom file to a XML document - * @param pom The path to the pom file - * @return The XML document - */ - private Document staticPomParse(Path pom) { - try { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - factory.setNamespaceAware(true); - factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); - DocumentBuilder builder = factory.newDocumentBuilder(); - return builder.parse(pom.toFile()); - } catch (Exception e) { - addError("Failed to parse pom file: " + pom, e); - raiseLastError(); - return null; + if (!pluginMetadata.getErrors().isEmpty()) { + LOG.debug("Precondition errors found for plugin {}", name); + pluginMetadata.save(); + return; } - } - /** - * Statically check the pom to ensure we can perform a modernization - * Some plugins are so old they don't support Java 8 or recent version of maven - * @param document The XML document of the pom - */ - private void checkStaticPom(Document document) { - try { - XPathFactory xPathFactory = XPathFactory.newInstance(); - XPath xpath = xPathFactory.newXPath(); - - // Check for parent relative path - Double parentRelativePath = (Double) xpath.evaluate( - "count(//*[local-name()='project']/*[local-name()='parent']/*[local-name()='relativePath'])", - document, - XPathConstants.NUMBER); - if (parentRelativePath == null || parentRelativePath.equals(0.0)) { - addError("Missing relativePath in parent pom"); - raiseLastError(); - return; - } - - // Check for old java.level - String javaVersion = (String) xpath.evaluate( - "//*[local-name()='project']/*[local-name()='properties']/*[local-name()='java.level']", - document, - XPathConstants.STRING); - if (javaVersion != null && (javaVersion.equals("7") || javaVersion.equals("6"))) { - addError("Found java.level with value 6 or 7. Cannot modernize this plugin. Too old."); - raiseLastError(); - return; - } - - // Check all repositories are using HTTPS - Double nonHttpsRepositories = (Double) xpath.evaluate( - "count(//*[local-name()='project']/*[local-name()='repositories']/*[local-name()='repository']/*[local-name()='url' and not(starts-with(., 'https'))])", - document, - XPathConstants.NUMBER); - if (nonHttpsRepositories != null && !nonHttpsRepositories.equals(0.0)) { - addError("Found non-https repository URL in pom file"); - raiseLastError(); - return; - } - - } catch (Exception e) { - addError("Failed to check pom file", e); - raiseLastError(); - } + // Collect using OpenRewrite + maven.collectMetadata(this); } /** @@ -665,6 +624,39 @@ public void setMetadata(PluginMetadata metadata) { this.metadata = metadata; } + /** + * Build cache manager for this plugin + * @return Cache manager + */ + public CacheManager buildPluginTargetDirectoryCacheManager() { + return new CacheManager(Path.of(Settings.TEST_PLUGINS_DIRECTORY) + .resolve(getLocalRepository().resolve("target"))); + } + + /** + * Static parse of the pom file to a XML document + * @param pom The path to the pom file + * @return The XML document + */ + private Document staticPomParse(Path pom) { + if (pom == null || !pom.toFile().exists()) { + addError("No pom file found"); + raiseLastError(); + return null; + } + try { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(true); + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + DocumentBuilder builder = factory.newDocumentBuilder(); + return builder.parse(pom.toFile()); + } catch (Exception e) { + addError("Failed to parse pom file: " + pom, e); + raiseLastError(); + return null; + } + } + @Override public String toString() { return name; diff --git a/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/model/PreconditionError.java b/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/model/PreconditionError.java new file mode 100644 index 00000000..e01b6238 --- /dev/null +++ b/plugin-modernizer-core/src/main/java/io/jenkins/tools/pluginmodernizer/core/model/PreconditionError.java @@ -0,0 +1,119 @@ +package io.jenkins.tools.pluginmodernizer.core.model; + +import java.util.function.BiFunction; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import org.w3c.dom.Document; + +/** + * Enum to represent the precondition errors preventing any modernization process + * Generally, these are the errors that need to be fixed before applying any modernization (very old plugin) + * We can provide in future version a way to fix these errors automatically (without OpenRewrite) by adding a fix function + * on this enum + */ +public enum PreconditionError { + + /** + * No pom file found + */ + NO_POM( + (document, xpath) -> { + return document == null; + }, + "No pom file found"), + + /** + * If the plugin has HTTP repositories preventing modernization + */ + MAVEN_REPOSITORIES_HTTP( + (document, xpath) -> { + if (document == null) { + return false; + } + try { + Double nonHttpsRepositories = (Double) xpath.evaluate( + "count(//*[local-name()='project']/*[local-name()='repositories']/*[local-name()='repository']/*[local-name()='url' and not(starts-with(., 'https'))])", + document, + XPathConstants.NUMBER); + return nonHttpsRepositories != null && !nonHttpsRepositories.equals(0.0); + } catch (Exception e) { + return false; + } + }, + "Found non-https repository URL in pom file preventing maven older than 3.8.1"), + + /** + * If the plugin has an older Java version preventing modernization + */ + OLDER_JAVA_VERSION( + (document, xpath) -> { + if (document == null) { + return false; + } + try { + String javaVersion = (String) xpath.evaluate( + "//*[local-name()='project']/*[local-name()='properties']/*[local-name()='java.level']", + document, + XPathConstants.STRING); + return javaVersion != null + && (javaVersion.equals("7") || javaVersion.equals("6") || javaVersion.equals("5")); + } catch (Exception e) { + return false; + } + }, + "Found older Java version in pom file preventing using recent Maven older than 3.9.x"), + + /** + * If the plugin has missing relative path preventing modernization + */ + MISSING_RELATIVE_PATH( + (document, xpath) -> { + try { + Double parentRelativePath = (Double) xpath.evaluate( + "count(//*[local-name()='project']/*[local-name()='parent']/*[local-name()='relativePath'])", + document, + XPathConstants.NUMBER); + return parentRelativePath == null || parentRelativePath.equals(0.0); + } catch (Exception e) { + return false; + } + }, + "Missing relative path in pom file preventing parent download"); + + /** + * Predicate to check if the flag is applicable for the given Document and XPath + */ + private final BiFunction isApplicable; + + /** + * Error message + */ + private final String error; + + /** + * Constructor + * @param isApplicable Predicate to check if the flag is applicable for the given XML document + */ + PreconditionError(BiFunction isApplicable, String error) { + this.isApplicable = isApplicable; + this.error = error; + } + + /** + * Check if the flag is applicable for the given Document and XPath + * @param Document the XML document + * @param xpath the XPath object + * @return true if the flag is applicable, false otherwise + */ + public boolean isApplicable(Document Document, XPath xpath) { + return isApplicable.apply(Document, xpath); + } + + /** + * Get the error message + * @return the error message + */ + public String getError() { + return error; + } +} diff --git a/plugin-modernizer-core/src/test/java/io/jenkins/tools/pluginmodernizer/core/extractor/MetadataCollectorTest.java b/plugin-modernizer-core/src/test/java/io/jenkins/tools/pluginmodernizer/core/extractor/MetadataCollectorTest.java index 2433b639..9ba8c613 100644 --- a/plugin-modernizer-core/src/test/java/io/jenkins/tools/pluginmodernizer/core/extractor/MetadataCollectorTest.java +++ b/plugin-modernizer-core/src/test/java/io/jenkins/tools/pluginmodernizer/core/extractor/MetadataCollectorTest.java @@ -8,8 +8,8 @@ import static org.openrewrite.maven.Assertions.pomXml; import io.jenkins.tools.pluginmodernizer.core.model.JDK; -import java.util.List; import java.util.Map; +import java.util.Set; import org.intellij.lang.annotations.Language; import org.junit.jupiter.api.Test; import org.openrewrite.test.RecipeSpec; @@ -169,7 +169,7 @@ void testPluginWithJenkinsfileWithoutJdkInfo() throws Exception { // Absent assertFalse(pluginMetadata.hasFile(ArchetypeCommonFile.WORKFLOW_CD)); - List jdkVersion = pluginMetadata.getJdks(); + Set jdkVersion = pluginMetadata.getJdks(); assertEquals(0, jdkVersion.size()); } @@ -193,7 +193,7 @@ void testPluginWithJenkinsfileWithJdkInfo() { assertTrue(pluginMetadata.hasFile(ArchetypeCommonFile.JENKINSFILE)); assertTrue(pluginMetadata.hasFile(ArchetypeCommonFile.POM)); - List jdkVersion = pluginMetadata.getJdks(); + Set jdkVersion = pluginMetadata.getJdks(); assertEquals(2, jdkVersion.size()); assertTrue(jdkVersion.contains(JDK.JAVA_21)); @@ -225,7 +225,7 @@ void testJenkinsfileWithConfigurationsAsParameter() { pomXml(POM_XML)); PluginMetadata pluginMetadata = new PluginMetadata().refresh(); assertTrue(pluginMetadata.hasFile(ArchetypeCommonFile.JENKINSFILE)); - List jdkVersion = pluginMetadata.getJdks(); + Set jdkVersion = pluginMetadata.getJdks(); assertEquals(2, jdkVersion.size()); } @@ -251,7 +251,7 @@ void testJenkinsfileWithInlineConfigurations() { pomXml(POM_XML)); PluginMetadata pluginMetadata = new PluginMetadata().refresh(); assertTrue(pluginMetadata.hasFile(ArchetypeCommonFile.JENKINSFILE)); - List jdkVersion = pluginMetadata.getJdks(); + Set jdkVersion = pluginMetadata.getJdks(); assertEquals(2, jdkVersion.size()); } }