diff --git a/src/main/java/org/sonarsource/scanner/maven/bootstrap/MavenProjectConverter.java b/src/main/java/org/sonarsource/scanner/maven/bootstrap/MavenProjectConverter.java index ea5eec89..ed7dd66e 100644 --- a/src/main/java/org/sonarsource/scanner/maven/bootstrap/MavenProjectConverter.java +++ b/src/main/java/org/sonarsource/scanner/maven/bootstrap/MavenProjectConverter.java @@ -89,9 +89,9 @@ public class MavenProjectConverter { public static final String FINDBUGS_EXCLUDE_FILTERS = "sonar.findbugs.excludeFilters"; - private static final String JAVA_PROJECT_MAIN_BINARY_DIRS = "sonar.java.binaries"; + public static final String JAVA_PROJECT_MAIN_BINARY_DIRS = "sonar.java.binaries"; - private static final String JAVA_PROJECT_MAIN_LIBRARIES = "sonar.java.libraries"; + public static final String JAVA_PROJECT_MAIN_LIBRARIES = "sonar.java.libraries"; private static final String SONAR_JAVA_JDK_HOME_PROPERTY = "sonar.java.jdkHome"; diff --git a/src/main/java/org/sonarsource/scanner/maven/bootstrap/ScannerBootstrapper.java b/src/main/java/org/sonarsource/scanner/maven/bootstrap/ScannerBootstrapper.java index 6b724d1f..c7beeb6b 100644 --- a/src/main/java/org/sonarsource/scanner/maven/bootstrap/ScannerBootstrapper.java +++ b/src/main/java/org/sonarsource/scanner/maven/bootstrap/ScannerBootstrapper.java @@ -132,7 +132,7 @@ Map collectProperties() } else if (mavenProjectConverter.isTestDirsOverridden()) { log.warn(notCollectingAdditionalSourcesBecauseOf(ScanProperties.PROJECT_TEST_DIRS)); } else { - collectAllSources(props); + collectAllSources(props, userProperties); } } @@ -162,7 +162,7 @@ private static Set excludedReportFiles(Map props) { } @VisibleForTesting - void collectAllSources(Map props) { + void collectAllSources(Map props, Properties userProperties) { String projectBasedir = props.get(ScanProperties.PROJECT_BASEDIR); // Exclude the files and folders covered by sonar.sources and sonar.tests (and sonar.exclusions) as computed by the MavenConverter // Combine all the sonar.sources at the top-level and by module @@ -178,9 +178,8 @@ void collectAllSources(Map props) { Set existingSources = coveredSources.stream() .map(Paths::get) .collect(Collectors.toSet()); - Set excludedFiles = excludedReportFiles(props); - - SourceCollector visitor = new SourceCollector(existingSources, mavenProjectConverter.getSkippedBasedDirs(), excludedFiles); + SourceCollector visitor = new SourceCollector(existingSources, mavenProjectConverter.getSkippedBasedDirs(), + excludedReportFiles(props), isUserDefinedJavaBinaries(userProperties)); Files.walkFileTree(Paths.get(projectBasedir), visitor); collectedSources = visitor.getCollectedSources().stream() .map(file -> file.toAbsolutePath().toString()) @@ -194,6 +193,11 @@ void collectAllSources(Map props) { } } + private static boolean isUserDefinedJavaBinaries(Properties userProperties) { + return userProperties.containsKey(MavenProjectConverter.JAVA_PROJECT_MAIN_LIBRARIES) && + userProperties.containsKey(MavenProjectConverter.JAVA_PROJECT_MAIN_BINARY_DIRS); + } + private void checkSQVersion() { if (isVersionPriorTo("5.6")) { throw new UnsupportedOperationException(UNSUPPORTED_BELOW_SONARQUBE_56_MESSAGE); diff --git a/src/main/java/org/sonarsource/scanner/maven/bootstrap/SourceCollector.java b/src/main/java/org/sonarsource/scanner/maven/bootstrap/SourceCollector.java index 09504d5b..d9b5c747 100644 --- a/src/main/java/org/sonarsource/scanner/maven/bootstrap/SourceCollector.java +++ b/src/main/java/org/sonarsource/scanner/maven/bootstrap/SourceCollector.java @@ -28,6 +28,8 @@ import java.util.HashSet; import java.util.Locale; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.stream.StreamSupport; public class SourceCollector implements FileVisitor { @@ -44,42 +46,45 @@ public class SourceCollector implements FileVisitor { ) ); - private static final Set EXCLUDED_EXTENSIONS = new HashSet<>( - Arrays.asList( - "jar", - "war", - "class", - "ear", - "nar", - // Archives - "DS_Store", - "zip", - "7z", - "rar", - "gz", - "tar", - "xz", - // log - "log", - // temp files - "bak", - "tmp", - "swp", - // ide files - "iml", - "ipr", - "iws", - "nib", - "log", - "java", - "jav", - "kt", - "scala" - ) - ); + private static final Set EXCLUDED_EXTENSIONS_WITH_BINARIES = Stream.of( + ".jar", + ".war", + ".class", + ".ear", + ".nar", + // Archives + "..DS_Store", + ".zip", + ".7z", + ".rar", + ".gz", + ".tar", + ".xz", + // log + ".log", + // temp files + ".bak", + ".tmp", + ".swp", + // ide files + ".iml", + ".ipr", + ".iws", + ".nib", + ".log") + .map(ext -> ext.toLowerCase(Locale.ROOT)) + .collect(Collectors.toSet()); + + private static final Set EXCLUDED_EXTENSIONS_WITHOUT_BINARIES = Stream.concat(EXCLUDED_EXTENSIONS_WITH_BINARIES.stream(), Stream.of( + ".java", + ".jav", + ".kt")).map(ext -> ext.toLowerCase(Locale.ROOT)) + .collect(Collectors.toSet()); + private final Set existingSources; private final Set directoriesToIgnore; private final Set excludedFiles; + private final Set excludedExtensions; public Set getCollectedSources() { return collectedSources; @@ -87,11 +92,14 @@ public Set getCollectedSources() { private final Set collectedSources = new HashSet<>(); - public SourceCollector(Set existingSources, Set directoriesToIgnore, Set excludedFiles) { + public SourceCollector(Set existingSources, Set directoriesToIgnore, Set excludedFiles, + boolean isUserDefinedJavaBinaries) { this.existingSources = existingSources; this.directoriesToIgnore = directoriesToIgnore; this.excludedFiles = excludedFiles; + this.excludedExtensions = isUserDefinedJavaBinaries ? EXCLUDED_EXTENSIONS_WITH_BINARIES : EXCLUDED_EXTENSIONS_WITHOUT_BINARIES; } + @Override public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes basicFileAttributes) throws IOException { if ( @@ -120,12 +128,11 @@ private boolean isCoveredByExistingSources(Path path) { @Override public FileVisitResult visitFile(Path path, BasicFileAttributes basicFileAttributes) { - if ( - !excludedFiles.contains(path) && - EXCLUDED_EXTENSIONS.stream().noneMatch(ext -> path.toString().endsWith(ext)) && - existingSources.stream().noneMatch(path::equals) - ) { - collectedSources.add(path); + if (!excludedFiles.contains(path) && existingSources.stream().noneMatch(path::equals)) { + String lowerCaseFileName = path.getFileName().toString().toLowerCase(Locale.ROOT); + if (excludedExtensions.stream().noneMatch(lowerCaseFileName::endsWith)) { + collectedSources.add(path); + } } return FileVisitResult.CONTINUE; } diff --git a/src/test/java/org/sonarsource/scanner/maven/SonarQubeMojoTest.java b/src/test/java/org/sonarsource/scanner/maven/SonarQubeMojoTest.java index 0cc2c3a1..e81637b9 100644 --- a/src/test/java/org/sonarsource/scanner/maven/SonarQubeMojoTest.java +++ b/src/test/java/org/sonarsource/scanner/maven/SonarQubeMojoTest.java @@ -23,6 +23,7 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.file.Path; import java.util.Arrays; import java.util.Map; import java.util.Properties; @@ -104,6 +105,50 @@ public void shouldExportOverridenWarWebSource() throws Exception { + new File(baseDir, "src/main/java").getAbsolutePath())); } + @Test + public void project_with_java_files_not_in_src() throws Exception { + File baseDir = executeProject( + "project-with-java-files-not-in-src", + "sonar.maven.scanAll", "true"); + Set actualListOfSources = extractSonarSources("target/dump.properties", baseDir.toPath()); + assertThat(actualListOfSources).containsExactlyInAnyOrder( + "/pom.xml", "/src/main/java"); + } + + @Test + public void project_with_java_files_not_in_src_and_user_defined_binaries() throws Exception { + File baseDir = executeProject( + "project-with-java-files-not-in-src", + "sonar.maven.scanAll", "true", + "sonar.java.binaries", "target/classes", + "sonar.java.libraries", "target/lib/logger.jar"); + Set actualListOfSources = extractSonarSources("target/dump.properties", baseDir.toPath()); + assertThat(actualListOfSources).containsExactlyInAnyOrder( + "/pom.xml", "/src/main/java", "/Hello.java", "/Hello.kt"); + } + + @Test + public void project_with_java_files_not_in_src_and_incomplete_user_defined_libraries() throws Exception { + File baseDir = executeProject( + "project-with-java-files-not-in-src", + "sonar.maven.scanAll", "true", + "sonar.java.binaries", "target/classes"); + Set actualListOfSources = extractSonarSources("target/dump.properties", baseDir.toPath()); + assertThat(actualListOfSources).containsExactlyInAnyOrder( + "/pom.xml", "/src/main/java"); + } + + @Test + public void project_with_java_files_not_in_src_and_incomplete_user_defined_binaries() throws Exception { + File baseDir = executeProject( + "project-with-java-files-not-in-src", + "sonar.maven.scanAll", "true", + "sonar.java.libraries", "target/lib/logger.jar"); + Set actualListOfSources = extractSonarSources("target/dump.properties", baseDir.toPath()); + assertThat(actualListOfSources).containsExactlyInAnyOrder( + "/pom.xml", "/src/main/java"); + } + // MSONAR-113 @Test public void shouldExportSurefireReportsPath() throws Exception { @@ -130,19 +175,7 @@ public void reuse_findbugs_exclusions_from_reporting() throws IOException, Excep @Test public void exclude_report_paths_from_scanAll() throws Exception { File projectBarDir = executeProject("project-with-external-reports", "sonar.maven.scanAll", "true"); - - String sources = readProps("target/dump.properties") - .entrySet() - .stream() - .filter(e -> e.getKey().toString().equals("sonar.sources")) - .map(Map.Entry::getValue) - .findFirst() - .orElse(null); - - Set actualListOfSources = Arrays.stream(sources.split(",")) - .map(file -> file.replace(projectBarDir.toString(), "").replace("\\", "/")) - .collect(Collectors.toSet()); - + Set actualListOfSources = extractSonarSources("target/dump.properties", projectBarDir.toPath()); assertThat(actualListOfSources).containsExactlyInAnyOrder("/other.xml", "/pom.xml"); } @@ -193,11 +226,11 @@ private File executeProject(String projectName, String... properties) throws Exc } @SafeVarargs - private final void assertPropsContains(MapEntry... entries) throws FileNotFoundException, IOException { + private final void assertPropsContains(MapEntry... entries) throws IOException { assertThat(readProps("target/dump.properties")).contains(entries); } - private Map readProps(String filePath) throws FileNotFoundException, IOException { + private static Map readProps(String filePath) throws IOException { FileInputStream fis = null; try { File dump = new File(filePath); @@ -210,4 +243,17 @@ private Map readProps(String filePath) throws FileNotFoundExcept } } + private static Set extractSonarSources(String propertiesPath, Path projectBarDir) throws IOException { + String sources = readProps(propertiesPath) + .entrySet() + .stream() + .filter(e -> e.getKey().toString().equals("sonar.sources")) + .map(Map.Entry::getValue) + .findFirst() + .orElse(null); + + return Arrays.stream(sources.split(",")) + .map(file -> file.replace(projectBarDir.toString(), "").replace("\\", "/")) + .collect(Collectors.toSet()); + } } diff --git a/src/test/java/org/sonarsource/scanner/maven/bootstrap/ScannerBootstrapperTest.java b/src/test/java/org/sonarsource/scanner/maven/bootstrap/ScannerBootstrapperTest.java index 67946421..8dca7969 100644 --- a/src/test/java/org/sonarsource/scanner/maven/bootstrap/ScannerBootstrapperTest.java +++ b/src/test/java/org/sonarsource/scanner/maven/bootstrap/ScannerBootstrapperTest.java @@ -165,7 +165,7 @@ void scanAll_property_is_applied_by_default() throws MojoExecutionException { }); verify(log, times(1)).info("Parameter sonar.maven.scanAll is enabled. The scanner will attempt to collect additional sources."); - verify(scannerBootstrapper, times(1)).collectAllSources(any()); + verify(scannerBootstrapper, times(1)).collectAllSources(any(), any()); } @Test @@ -179,7 +179,7 @@ void scanAll_property_is_not_applied_when_set_explicitly() throws MojoExecutionE }); verify(log, never()).info("Parameter sonar.maven.scanAll is enabled. The scanner will attempt to collect additional sources."); - verify(scannerBootstrapper, never()).collectAllSources(any()); + verify(scannerBootstrapper, never()).collectAllSources(any(), any()); } @Test @@ -194,7 +194,7 @@ void scanAll_property_is_applied_when_set_explicitly() throws MojoExecutionExcep }); verify(log, times(1)).info("Parameter sonar.maven.scanAll is enabled. The scanner will attempt to collect additional sources."); - verify(scannerBootstrapper, times(1)).collectAllSources(any()); + verify(scannerBootstrapper, times(1)).collectAllSources(any(), any()); } @Test @@ -212,7 +212,7 @@ void should_not_collect_all_sources_when_sonar_sources_is_overridden() throws Mo verify(log, times(1)).info("Parameter sonar.maven.scanAll is enabled. The scanner will attempt to collect additional sources."); verify(log, times(1)).warn("Parameter sonar.maven.scanAll is enabled but the scanner will not collect additional sources because sonar.sources has been overridden."); - verify(scannerBootstrapper, never()).collectAllSources(any()); + verify(scannerBootstrapper, never()).collectAllSources(any(), any()); } @Test @@ -231,7 +231,7 @@ void should_not_collect_all_sources_when_sonar_tests_is_overridden() throws Mojo verify(log, times(1)).info("Parameter sonar.maven.scanAll is enabled. The scanner will attempt to collect additional sources."); verify(log, times(1)).warn("Parameter sonar.maven.scanAll is enabled but the scanner will not collect additional sources because sonar.tests has been overridden."); - verify(scannerBootstrapper, never()).collectAllSources(any()); + verify(scannerBootstrapper, never()).collectAllSources(any(), any()); } @Test diff --git a/src/test/java/org/sonarsource/scanner/maven/bootstrap/SourceCollectorTest.java b/src/test/java/org/sonarsource/scanner/maven/bootstrap/SourceCollectorTest.java index a7dd6d47..cb8c422e 100644 --- a/src/test/java/org/sonarsource/scanner/maven/bootstrap/SourceCollectorTest.java +++ b/src/test/java/org/sonarsource/scanner/maven/bootstrap/SourceCollectorTest.java @@ -55,7 +55,7 @@ static void setup() throws IOException { void testPrevisitDirectories() throws IOException { Path srcMainJava = Paths.get("src", "main", "java"); Set existingSources = Collections.singleton(srcMainJava); - FileVisitor visitor = new SourceCollector(existingSources, Collections.emptySet(), Collections.emptySet()); + FileVisitor visitor = new SourceCollector(existingSources, Collections.emptySet(), Collections.emptySet(), false); Path gitFolder = Paths.get(".git"); @@ -82,15 +82,15 @@ void testPrevisitDirectories() throws IOException { @Test void visitorCollectsConsistently() throws IOException { // File in the existing source is not repeated in the collected files - SourceCollector visitor = new SourceCollector(Collections.emptySet(), Collections.emptySet(), Collections.emptySet()); + SourceCollector visitor = new SourceCollector(Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), false); Files.walkFileTree(emptyProjectBasedir, visitor); assertThat(visitor.getCollectedSources()).isEmpty(); - SourceCollector otherVisitor = new SourceCollector(Collections.emptySet(), Collections.emptySet(), Collections.emptySet()); + SourceCollector otherVisitor = new SourceCollector(Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), false); Files.walkFileTree(singleFileProjectBaseDir, otherVisitor); assertThat(otherVisitor.getCollectedSources()).containsOnly(singleFileProjectBaseDir.resolve("pom.xml")); - SourceCollector visitorAvoidingPomXml = new SourceCollector(Collections.singleton(singleFileProjectBaseDir.resolve("pom.xml")), Collections.emptySet(), Collections.emptySet()); + SourceCollector visitorAvoidingPomXml = new SourceCollector(Collections.singleton(singleFileProjectBaseDir.resolve("pom.xml")), Collections.emptySet(), Collections.emptySet(), false); Files.walkFileTree(singleFileProjectBaseDir, visitorAvoidingPomXml); assertThat(visitorAvoidingPomXml.getCollectedSources()).isEmpty(); } @@ -104,7 +104,7 @@ void visitorIgnoresFilesInDirectoriesToIgnore() throws IOException { Path fileInSubModule = subModule.resolve("ignore-me.php"); fileInSubModule.toFile().createNewFile(); - SourceCollector visitor = new SourceCollector(Collections.emptySet(), Collections.singleton(subModule), Collections.emptySet()); + SourceCollector visitor = new SourceCollector(Collections.emptySet(), Collections.singleton(subModule), Collections.emptySet(), false); Files.walkFileTree(simpleProjectBasedDir, visitor); assertThat(visitor.getCollectedSources()).doesNotContain(fileInSubModule); } diff --git a/src/test/projects/project-with-java-files-not-in-src/Hello.java b/src/test/projects/project-with-java-files-not-in-src/Hello.java new file mode 100644 index 00000000..72342c80 --- /dev/null +++ b/src/test/projects/project-with-java-files-not-in-src/Hello.java @@ -0,0 +1,5 @@ +public class Hello { + public static void main (String[] args) { + System.out.println("Hello World!"); + } +} diff --git a/src/test/projects/project-with-java-files-not-in-src/Hello.kt b/src/test/projects/project-with-java-files-not-in-src/Hello.kt new file mode 100644 index 00000000..51a1d567 --- /dev/null +++ b/src/test/projects/project-with-java-files-not-in-src/Hello.kt @@ -0,0 +1,3 @@ +fun main(args : Array) { + println("Hello, World!") +} diff --git a/src/test/projects/project-with-java-files-not-in-src/pom.xml b/src/test/projects/project-with-java-files-not-in-src/pom.xml new file mode 100644 index 00000000..bae60ec6 --- /dev/null +++ b/src/test/projects/project-with-java-files-not-in-src/pom.xml @@ -0,0 +1,14 @@ + + 4.0.0 + + org.codehaus.sonar + project-with-java-files-not-in-src + 1.0-SNAPSHOT + + + target/dump.properties + + + diff --git a/src/test/projects/project-with-java-files-not-in-src/src/main/java/Main.java b/src/test/projects/project-with-java-files-not-in-src/src/main/java/Main.java new file mode 100644 index 00000000..f88604e4 --- /dev/null +++ b/src/test/projects/project-with-java-files-not-in-src/src/main/java/Main.java @@ -0,0 +1,5 @@ +public class Main { + public static void main (String[] args) { + System.out.println("Hello World!"); + } +}