Merge pull request #23 from JetBrains/thresholds-by-severity

Add severity specific fail thresholds

baseline-cli/src/main/kotlin/BaselineCli.kt

@@ -1,16 +1,18 @@
+import Severity.Companion.severity
 import com.google.gson.JsonSyntaxException
 import com.jetbrains.qodana.sarif.RuleUtil
 import com.jetbrains.qodana.sarif.SarifUtil
 import com.jetbrains.qodana.sarif.baseline.BaselineCalculation
 import com.jetbrains.qodana.sarif.model.Invocation
+import com.jetbrains.qodana.sarif.model.Result
 import com.jetbrains.qodana.sarif.model.Run
 import com.jetbrains.qodana.sarif.model.SarifReport
 import java.net.URI
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
 
-object BaselineCli {
+internal object BaselineCli {
     fun process(options: BaselineOptions, cliPrinter: (String) -> Unit, errPrinter: (String) -> Unit): Int {
         if (!Files.exists(Paths.get(options.sarifPath))) {
             errPrinter("Please provide a valid SARIF report path")
@@ -32,53 +34,69 @@ object BaselineCli {
                 sarifReport,
                 Paths.get(options.sarifPath),
                 Paths.get(options.baselinePath),
-                options.failThreshold,
+                options.thresholds,
                 options.includeAbsent,
                 printer,
                 cliPrinter,
                 errPrinter
             )
         } else {
-            compareThreshold(sarifReport, Paths.get(options.sarifPath), options.failThreshold, printer, cliPrinter, errPrinter)
+            compareThreshold(
+                sarifReport,
+                Paths.get(options.sarifPath),
+                options.thresholds,
+                printer,
+                cliPrinter,
+                errPrinter
+            )
         }
     }
 
     private fun processResultCount(
-        size: Int,
-        failThreshold: Int?,
+        results: List<Result>?,
+        hasBaseline: Boolean,
+        thresholds: SeverityThresholds?,
         cliPrinter: (String) -> Unit,
         errPrinter: (String) -> Unit
     ): Invocation {
+        val size = results?.size ?: 0
         if (size > 0) {
             errPrinter("Found $size new problems according to the checks applied")
         } else {
             cliPrinter("It seems all right \uD83D\uDC4C No new problems found according to the checks applied")
         }
-        if (failThreshold != null && size > failThreshold) {
-            errPrinter("New problems count $size is greater than the threshold $failThreshold")
+        val failedThresholds = thresholds?.let { checkSeverityThresholds(results, hasBaseline, it) }
+
+        return if (failedThresholds.isNullOrEmpty()) {
+            Invocation().apply {
+                exitCode = 0
+                executionSuccessful = true
+            }
+        } else {
+            val msg = buildString {
+                append(if (failedThresholds.size == 1) "Failure condition triggered" else "Failure conditions triggered")
+                failedThresholds.joinTo(buffer = this, separator = "\n - ", prefix = "\n - " )
+            }
+            errPrinter(msg)
             return Invocation().apply {
                 exitCode = THRESHOLD_EXIT
-                exitCodeDescription = "Qodana reached failThreshold"
+                exitCodeDescription = msg
                 executionSuccessful = true
             }
         }
-        return Invocation().apply {
-            exitCode = 0
-            executionSuccessful = true
-        }
     }
 
     private fun compareThreshold(
         sarifReport: SarifReport,
         sarifPath: Path,
-        failThreshold: Int?,
+        thresholds: SeverityThresholds?,
         printer: CommandLineResultsPrinter,
         cliPrinter: (String) -> Unit,
         errPrinter: (String) -> Unit
     ): Int {
         val results = sarifReport.runs.first().results
         printer.printResults(results, "Qodana - Detailed summary")
-        val invocation = processResultCount(results.size, failThreshold, cliPrinter, errPrinter)
+        val invocation = processResultCount(results, false, thresholds, cliPrinter, errPrinter)
         sarifReport.runs.first().invocations = listOf(invocation)
         SarifUtil.writeReport(sarifPath, sarifReport)
         return invocation.exitCode
@@ -88,7 +106,7 @@ object BaselineCli {
         sarifReport: SarifReport,
         sarifPath: Path,
         baselinePath: Path,
-        failThreshold: Int?,
+        thresholds: SeverityThresholds?,
         includeAbsent: Boolean,
         printer: CommandLineResultsPrinter,
         cliPrinter: (String) -> Unit,
@@ -105,14 +123,56 @@ object BaselineCli {
             errPrinter("Error reading baseline report: ${e.message}")
             return ERROR_EXIT
         }
-        val baselineCalculation = BaselineCalculation.compare(sarifReport, baseline, BaselineCalculation.Options(includeAbsent))
-        printer.printResultsWithBaselineState(sarifReport.runs.first().results, includeAbsent)
-        val invocation = processResultCount(baselineCalculation.newResults, failThreshold, cliPrinter, errPrinter)
+        BaselineCalculation.compare(sarifReport, baseline, BaselineCalculation.Options(includeAbsent))
+        val results = sarifReport.runs.first().results
+        printer.printResultsWithBaselineState(results, includeAbsent)
+        val invocation = processResultCount(results, true, thresholds, cliPrinter, errPrinter)
         sarifReport.runs.first().invocations = listOf(invocation)
         SarifUtil.writeReport(sarifPath, sarifReport)
         return invocation.exitCode
     }
 
+    private fun checkSeverityThresholds(
+        results: List<Result>?,
+        hasBaseline: Boolean,
+        thresholds: SeverityThresholds
+    ): List<String> {
+        val baselineFilter: (Result) -> Boolean = when {
+            !hasBaseline -> { _ -> true }
+            else -> { x -> x.baselineState == Result.BaselineState.NEW }
+        }
+
+        val resultsBySeverity = results.orEmpty()
+            .asSequence()
+            .filter(baselineFilter)
+            .groupingBy { it.severity() }
+            .eachCount()
+
+        val failedSeverities = resultsBySeverity.asSequence()
+            .mapNotNull { (severity, count) ->
+                val threshold = thresholds.bySeverity(severity)
+                if (threshold != null && count > threshold) {
+                    Triple(severity, count, threshold)
+                } else {
+                    null
+                }
+            }
+            .sortedBy { (severity) -> severity }
+            .map { (severity, count, threshold) ->
+                val p = if (count == 1) "1 problem" else "$count problems"
+                "Detected $p for severity ${severity.name}, fail threshold $threshold"
+            }
+            .toList()
+
+        val total = resultsBySeverity.values.sum()
+        return if (thresholds.any != null && total > thresholds.any) {
+            val p = if (total == 1) "1 problem" else "$total problems"
+            failedSeverities + "Detected $p across all severities, fail threshold ${thresholds.any}"
+        } else {
+            failedSeverities
+        }
+    }
+
     private fun createSarifReport(runs: List<Run>): SarifReport {
         val schema =
             URI("https://raw.githubusercontent.com/schemastore/schemastore/master/src/schemas/json/sarif-2.1.0-rtm.5.json")
@@ -124,4 +184,5 @@ object BaselineCli {
 
         return { state.computeIfAbsent(it, f) }
     }
+
 }

baseline-cli/src/main/kotlin/Main.kt

@@ -1,5 +1,6 @@
 import BaselineCli.process
 import com.github.ajalt.clikt.core.CliktCommand
+import com.github.ajalt.clikt.parameters.options.NullableOption
 import com.github.ajalt.clikt.parameters.options.flag
 import com.github.ajalt.clikt.parameters.options.option
 import com.github.ajalt.clikt.parameters.options.required
@@ -13,19 +14,48 @@ class BaselineCommand : CliktCommand() {
     private val sarifReport: String by option("-r", help = "Sarif report path").required()
     private val baselineReport: String? by option("-b", help = "Baseline report path")
     private val baselineIncludeAbsent: Boolean by option("-i", help = "Baseline include absent status").flag()
-    private val failThreshold: Int? by option("-f", help = "Fail threshold").int()
+
+    private fun threshold(severity: Severity): NullableOption<Int, Int> {
+        val lc = severity.name.lowercase()
+        return option("--threshold-$lc", help = "Fail threshold for $lc severity").int()
+    }
+
+    private val failThresholdAny: Int? by option(
+        "-f",
+        "--threshold-any",
+        help = "Fail threshold for any severity"
+    ).int()
+
+    private val failThresholdCritical: Int? by threshold(Severity.CRITICAL)
+    private val failThresholdHigh: Int? by threshold(Severity.HIGH)
+    private val failThresholdModerate: Int? by threshold(Severity.MODERATE)
+    private val failThresholdLow: Int? by threshold(Severity.LOW)
+    private val failThresholdInfo: Int? by threshold(Severity.INFO)
+
+
     override fun run() {
+        val thresholds = SeverityThresholds(
+            any = failThresholdAny,
+            critical = failThresholdCritical,
+            high = failThresholdHigh,
+            moderate = failThresholdModerate,
+            low = failThresholdLow,
+            info = failThresholdInfo
+        )
         val ret = process(
-            BaselineOptions(sarifReport, baselineReport, failThreshold, baselineIncludeAbsent),
-            { println(it) },
-            { System.err.println(it) })
+            BaselineOptions(sarifReport, baselineReport, thresholds, baselineIncludeAbsent),
+            ::println,
+            System.err::println
+        )
         exitProcess(ret)
     }
 }
 
-data class BaselineOptions(val sarifPath: String,
-                           val baselinePath: String? = null,
-                           val failThreshold: Int? = null,
-                           val includeAbsent: Boolean = false)
+internal data class BaselineOptions(
+    val sarifPath: String,
+    val baselinePath: String? = null,
+    val thresholds: SeverityThresholds? = null,
+    val includeAbsent: Boolean = false,
+)
 
 fun main(args: Array<String>) = BaselineCommand().main(args)

baseline-cli/src/main/kotlin/Severity.kt

@@ -0,0 +1,60 @@
+import com.jetbrains.qodana.sarif.model.Level
+import com.jetbrains.qodana.sarif.model.Result
+
+
+internal data class SeverityThresholds(
+    val any: Int? = null,
+    val critical: Int? = null,
+    val high: Int? = null,
+    val moderate: Int? = null,
+    val low: Int? = null,
+    val info: Int? = null,
+) {
+    fun bySeverity(qodanaSeverity: Severity) = when (qodanaSeverity) {
+        Severity.INFO -> info
+        Severity.LOW -> low
+        Severity.MODERATE -> moderate
+        Severity.HIGH -> high
+        Severity.CRITICAL -> critical
+    }
+}
+
+internal enum class Severity {
+    CRITICAL,
+    HIGH,
+    MODERATE,
+    LOW,
+    INFO;
+
+    companion object {
+        fun Result.severity() =
+            (properties?.get("qodanaSeverity") as? String)?.let(::fromQodanaSeverity)
+                ?: (properties?.get("ideaSeverity") as? String)?.let(::fromIdeaSeverity)
+                ?: level?.let(::fromSarif)
+                ?: MODERATE
+
+        private fun fromQodanaSeverity(value: String): Severity? = when (value) {
+            "Critical" -> CRITICAL
+            "High" -> HIGH
+            "Moderate" -> MODERATE
+            "Low" -> LOW
+            "Info" -> INFO
+            else -> null
+        }
+
+        private fun fromIdeaSeverity(value: String): Severity = when (value) {
+            "ERROR" -> CRITICAL
+            "WARNING" -> HIGH
+            "WEAK_WARNING" -> MODERATE
+            "TYPO" -> LOW
+            else -> INFO
+        }
+
+        private fun fromSarif(level: Level): Severity = when (level) {
+            Level.ERROR -> CRITICAL
+            Level.WARNING -> HIGH
+            else -> MODERATE
+        }
+    }
+
+}

baseline-cli/src/test/kotlin/BaselineCliTest.kt

@@ -64,39 +64,62 @@ class BaselineCliTest {
     }
 
     @Test
-    fun `test when failThreshold is not present and results count is less than the failThreshold default value`() {
+    fun `test when failThreshold is provided and results count in sarifReport is more than failThreshold`() {
         // Act
         val exitCode = assertDoesNotThrow {
-            BaselineCli.process(BaselineOptions(sarif), stdout::append, stderr::append)
+            BaselineCli.process(BaselineOptions(sarif, thresholds = SeverityThresholds(any = 0)), stdout::append, stderr::append)
         }
 
         // Assert
-        assertEquals(0, exitCode)
-        assertTrue(!stdout.contains("is greater than the threshold"))
+        assertEquals(THRESHOLD_EXIT, exitCode)
+        assertTrue(stderr.contains("Detected 2 problems across all severities, fail threshold 0"))
     }
 
     @Test
-    fun `test when failThreshold is provided and results count in sarifReport is more than failThreshold`() {
+    fun `test when failThreshold is provided and results count in sarifReport is equal to failThreshold`() {
         // Act
         val exitCode = assertDoesNotThrow {
-            BaselineCli.process(BaselineOptions(sarif, failThreshold = 0), stdout::append, stderr::append)
+            BaselineCli.process(BaselineOptions(sarif, thresholds = SeverityThresholds(any = 2)), stdout::append, stderr::append)
+        }
+
+        // Assert
+        assertEquals(0, exitCode)
+        assertTrue(stderr.contains("Found 2 new problems according to the checks applied"))
+    }
+
+    @Test
+    fun `test when fail threshold for severity is provided and results count exceeds threshold`() {
+        val exitCode = assertDoesNotThrow {
+            BaselineCli.process(BaselineOptions(sarif, thresholds = SeverityThresholds(high = 1)), stdout::append, stderr::append)
         }
 
         // Assert
         assertEquals(THRESHOLD_EXIT, exitCode)
-        assertTrue(stderr.contains("New problems count") && stderr.contains("is greater than the threshold"))
+        assertTrue(stderr.contains("Detected 2 problems for severity HIGH, fail threshold 1"))
+    }
+
+    @Test
+    fun `test when fail threshold for severity is provided and results count does not exceed threshold`() {
+        val exitCode = assertDoesNotThrow {
+            BaselineCli.process(BaselineOptions(sarif, thresholds = SeverityThresholds(low = 2)), stdout::append, stderr::append)
+        }
+
+        // Assert
+        assertEquals(0, exitCode)
+        assertFalse(stderr.contains("LOW"))
+        assertTrue(stderr.contains("Found 2 new problems according to the checks applied"))
     }
 
     @Test
     fun `test when failThreshold is provided and newResults count in baselineCalculation is more than failThreshold`() {
         // Act
         val exitCode = assertDoesNotThrow {
-            BaselineCli.process(BaselineOptions(sarif, emptySarif, failThreshold = 1), stdout::append, stderr::append)
+            BaselineCli.process(BaselineOptions(sarif, emptySarif, thresholds = SeverityThresholds(any = 1)), stdout::append, stderr::append)
         }
 
         // Assert
         assertEquals(THRESHOLD_EXIT, exitCode)
-        assertTrue(stderr.contains("New problems count") && stderr.contains("is greater than the threshold"))
+        assertTrue(stderr.contains("Detected 2 problems across all severities, fail threshold 1"))
     }
 
     @Test