Add basic CLI tests.

CHANGELOG.md

@@ -6,6 +6,7 @@ This project adheres to [Semantic Versioning](https://semver.org/).
 
 ### Added
 - Use Gradle JVM toolchain with language version 8 to compile the project
+- Basic tests for CLI ([#540](https://github.com/pinterest/ktlint/issues/540))
 
 ### Fixed
 - Fix false positive in rule spacing-between-declarations-with-annotations ([#1281](https://github.com/pinterest/ktlint/issues/1281))

build.gradle

@@ -17,6 +17,9 @@ ext.deps = [
   'picocli'       : 'info.picocli:picocli:3.9.6',
   // Testing libraries
   'junit'         : 'junit:junit:4.13.1',
+  'junit5Api'     : 'org.junit.jupiter:junit-jupiter-api:5.8.2',
+  'junit5Jupiter' : 'org.junit.jupiter:junit-jupiter-engine:5.8.2',
+  'junit5Vintage' : 'org.junit.vintage:junit-vintage-engine:5.8.2',
   'assertj'       : 'org.assertj:assertj-core:3.12.2',
   'sarif4k'       : 'io.github.detekt.sarif4k:sarif4k:0.0.1',
   'jimfs'         : 'com.google.jimfs:jimfs:1.1'
@@ -45,7 +48,7 @@ task ktlint(type: JavaExec, group: LifecycleBasePlugin.VERIFICATION_GROUP) {
   description = "Check Kotlin code style."
   classpath = configurations.ktlint
   main = 'com.pinterest.ktlint.Main'
-  args '**/src/**/*.kt', '--baseline=ktlint-baseline.xml', '--verbose'
+  args '**/src/**/*.kt', '!**/resources/cli/**', '--baseline=ktlint-baseline.xml', '--verbose'
 }
 
 /**

gradle/verification-metadata.xml

@@ -128,6 +128,8 @@
             <trusting group="org.junit.jupiter"/>
             <trusting group="junit"/>
             <trusting group="org.junit"/>
+            <trusting group="org.junit.vintage"/>
+            <trusting group="org.apiguardian"/>
          </trusted-key>
       </trusted-keys>
    </configuration>

ktlint/build.gradle

@@ -35,8 +35,12 @@ dependencies {
   implementation deps.picocli
 
   testImplementation deps.junit
+  testImplementation deps.junit5Api
   testImplementation deps.assertj
   testImplementation deps.jimfs
+
+  testRuntimeOnly deps.junit5Jupiter
+  testRuntimeOnly deps.junit5Vintage
 }
 
 // Implements https://github.com/brianm/really-executable-jars-maven-plugin maven plugin behaviour.
@@ -75,3 +79,19 @@ tasks.register("shadowJarExecutableChecksum", Checksum.class) {
 
   algorithm = Checksum.Algorithm.MD5
 }
+
+tasks.withType(Test).configureEach {
+  it.dependsOn(shadowJarExecutableTask)
+  it.useJUnitPlatform()
+
+  doFirst {
+    it.systemProperty(
+      "ktlint-cli",
+      shadowJarExecutableTask.get().outputs.files.find { it.name == "ktlint" }.absolutePath
+    )
+    it.systemProperty(
+      "ktlint-version",
+      version
+    )
+  }
+}

ktlint/src/test/kotlin/com/pinterest/ktlint/BaseCLITest.kt

@@ -0,0 +1,116 @@
+package com.pinterest.ktlint
+
+import java.io.File
+import java.nio.file.FileVisitResult
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.Paths
+import java.nio.file.SimpleFileVisitor
+import java.nio.file.attribute.BasicFileAttributes
+import java.util.concurrent.TimeUnit
+import org.junit.jupiter.api.io.TempDir
+
+abstract class BaseCLITest {
+    private val ktlintCli: String = System.getProperty("ktlint-cli")
+
+    @TempDir
+    private lateinit var tempDir: Path
+
+    fun runKtLintCliProcess(
+        testProjectName: String,
+        arguments: List<String> = emptyList(),
+        executionAssertions: ExecutionResult.() -> Unit
+    ) {
+        val projectPath = prepareTestProject(testProjectName)
+        val ktlintCommand = "$ktlintCli ${arguments.joinToString()}"
+        // Forking in a new shell process, so 'ktlint' will pickup new 'PATH' env variable value
+        val pb = ProcessBuilder("/bin/sh", "-c", ktlintCommand)
+        pb.directory(projectPath.toAbsolutePath().toFile())
+
+        // Overriding user path to java executable to use java version test is running on
+        val environment = pb.environment()
+        environment["PATH"] = "${System.getProperty("java.home")}${File.separator}bin${File.pathSeparator}${System.getenv()["PATH"]}"
+
+        val process = pb.start()
+        val output = process.inputStream.bufferedReader().use { it.readLines() }
+        val error = process.errorStream.bufferedReader().use { it.readLines() }
+        process.waitFor(WAIT_TIME_SEC, TimeUnit.SECONDS)
+
+        executionAssertions(ExecutionResult(process.exitValue(), output, error, projectPath))
+
+        process.destroy()
+    }
+
+    private fun prepareTestProject(testProjectName: String): Path {
+        val testProjectPath = testProjectsPath.resolve(testProjectName)
+        assert(Files.exists(testProjectPath)) {
+            "Test project $testProjectName does not exist!"
+        }
+
+        return tempDir.resolve(testProjectName).also { testProjectPath.copyRecursively(it) }
+    }
+
+    private fun Path.copyRecursively(dest: Path) {
+        Files.walkFileTree(
+            this,
+            object : SimpleFileVisitor<Path>() {
+                override fun preVisitDirectory(
+                    dir: Path,
+                    attrs: BasicFileAttributes
+                ): FileVisitResult {
+                    Files.createDirectories(dest.resolve(relativize(dir)))
+                    return FileVisitResult.CONTINUE
+                }
+
+                override fun visitFile(
+                    file: Path,
+                    attrs: BasicFileAttributes
+                ): FileVisitResult {
+                    Files.copy(file, dest.resolve(relativize(file)))
+                    return FileVisitResult.CONTINUE
+                }
+            }
+        )
+    }
+
+    data class ExecutionResult(
+        val exitCode: Int,
+        val normalOutput: List<String>,
+        val errorOutput: List<String>,
+        val testProject: Path
+    ) {
+        fun assertNormalExitCode() {
+            assert(exitCode == 0) {
+                "Execution was not finished normally: $exitCode"
+            }
+        }
+
+        fun assertErrorExitCode() {
+            assert(exitCode == 1) {
+                "Execution was finished without error: $exitCode"
+            }
+        }
+
+        fun assertErrorOutputIsEmpty() {
+            assert(errorOutput.isEmpty()) {
+                "Error output contains following lines:\n${errorOutput.joinToString(separator = "\n")}"
+            }
+        }
+
+        fun assertSourceFileWasFormatted(
+            filePathInProject: String
+        ) {
+            val originalFile = testProjectsPath.resolve(testProject.last()).resolve(filePathInProject)
+            val newFile = testProject.resolve(filePathInProject)
+
+            assert(originalFile.toFile().readText() != newFile.toFile().readText()) {
+                "Format did not change source file $filePathInProject content:\n${originalFile.toFile().readText()}"
+            }
+        }
+    }
+
+    companion object {
+        private const val WAIT_TIME_SEC = 3L
+        val testProjectsPath: Path = Paths.get("src", "test", "resources", "cli")
+    }
+}

ktlint/src/test/kotlin/com/pinterest/ktlint/SimpleCLITest.kt

@@ -0,0 +1,84 @@
+package com.pinterest.ktlint
+
+import org.junit.jupiter.api.DisplayName
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.condition.DisabledOnOs
+import org.junit.jupiter.api.condition.OS
+
+@DisabledOnOs(OS.WINDOWS)
+@DisplayName("CLI basic checks")
+class SimpleCLITest : BaseCLITest() {
+
+    @DisplayName("Should print help")
+    @Test
+    fun shouldOutputHelp() {
+        runKtLintCliProcess(
+            "no-code-style-error",
+            listOf("--help")
+        ) {
+            assertNormalExitCode()
+            assertErrorOutputIsEmpty()
+
+            assert(normalOutput.contains("Usage:") && normalOutput.contains("Examples:")) {
+                "Did not produced help output!\n ${normalOutput.joinToString(separator = "\n")}"
+            }
+        }
+    }
+
+    @DisplayName("Should print correct version")
+    @Test
+    fun shouldCorrectlyPrintVersion() {
+        runKtLintCliProcess(
+            "no-code-style-error",
+            listOf("--version")
+        ) {
+            assertNormalExitCode()
+            assertErrorOutputIsEmpty()
+
+            val expectedVersion = System.getProperty("ktlint-version")
+            assert(normalOutput.contains(expectedVersion)) {
+                "Output did not contain expected $expectedVersion version:\n ${normalOutput.joinToString(separator = "\n")}"
+            }
+        }
+    }
+
+    @DisplayName("Should complete lint without errors")
+    @Test
+    internal fun lintWithoutErrors() {
+        runKtLintCliProcess(
+            "no-code-style-error"
+        ) {
+            assertNormalExitCode()
+            assertErrorOutputIsEmpty()
+        }
+    }
+
+    @DisplayName("Should complete lint with error")
+    @Test
+    internal fun lintWithError() {
+        runKtLintCliProcess(
+            "too-many-empty-lines"
+        ) {
+            assertErrorExitCode()
+
+            assert(normalOutput.find { it.contains("Needless blank line(s)") } != null) {
+                "Unexpected output:\n${normalOutput.joinToString(separator = "\n")}"
+            }
+        }
+    }
+
+    @DisplayName("Should format without errors")
+    @Test
+    internal fun formatWorks() {
+        runKtLintCliProcess(
+            "too-many-empty-lines",
+            listOf("-F")
+        ) {
+            assertNormalExitCode()
+            // on JDK11+ contains warning about illegal reflective access operation
+            // assertErrorOutputIsEmpty()
+
+            assertSourceFileWasFormatted("main.kt")
+        }
+    }
+}

ktlint/src/test/resources/cli/no-code-style-error/main.kt

@@ -0,0 +1,5 @@
+package test
+
+fun main() {
+    println("Hello world!")
+}

ktlint/src/test/resources/cli/too-many-empty-lines/main.kt

@@ -0,0 +1,8 @@
+package test
+
+fun main() {
+
+
+
+    println("Hello world!")
+}