diff --git a/application/src/main/kotlin/application/ExamplesCommand.kt b/application/src/main/kotlin/application/ExamplesCommand.kt index 42e5340db..31dff990a 100644 --- a/application/src/main/kotlin/application/ExamplesCommand.kt +++ b/application/src/main/kotlin/application/ExamplesCommand.kt @@ -1,12 +1,13 @@ package application +import io.specmatic.core.Feature import io.specmatic.core.Result import io.specmatic.core.Results import io.specmatic.core.SPECMATIC_STUB_DICTIONARY import io.specmatic.core.examples.server.ExamplesInteractiveServer import io.specmatic.core.examples.server.ExamplesInteractiveServer.Companion.validateSingleExample +import io.specmatic.core.examples.server.defaultExternalExampleDirFrom import io.specmatic.core.examples.server.loadExternalExamples -import io.specmatic.core.filters.ScenarioMetadataFilter import io.specmatic.core.log.* import io.specmatic.core.parseContractFileToFeature import io.specmatic.core.pattern.ContractException @@ -16,6 +17,10 @@ import picocli.CommandLine.* import java.io.File import java.lang.Thread.sleep import java.util.concurrent.Callable +import kotlin.system.exitProcess + +private const val SUCCESS_EXIT_CODE = 0 +private const val FAILURE_EXIT_CODE = 1 @Command( name = "examples", @@ -99,11 +104,11 @@ For example: override fun call(): Int { if (contractFile == null) { println("No contract file provided. Use a subcommand or provide a contract file. Use --help for more details.") - return 1 + return FAILURE_EXIT_CODE } if (!contractFile!!.exists()) { logger.log("Could not find file ${contractFile!!.path}") - return 1 + return FAILURE_EXIT_CODE } configureLogger(this.verbose) @@ -120,10 +125,10 @@ For example: ) } catch (e: Throwable) { logger.log(e) - return 1 + return FAILURE_EXIT_CODE } - return 0 + return SUCCESS_EXIT_CODE } @Command( @@ -174,12 +179,25 @@ For example: ) var filterNot: List = emptyList() - @Option(names = ["--contract-file"], description = ["Contract file path"], required = true) - lateinit var contractFile: File + @Option(names = ["--contract-file", "--spec-file"], description = ["Contract file path"], required = false) + var contractFile: File? = null @Option(names = ["--example-file"], description = ["Example file path"], required = false) val exampleFile: File? = null + @Option(names = ["--examples-dir"], description = ["External examples directory path for a single API specification (If you are not following the default naming convention for external examples directory)"], required = false) + val examplesDir: File? = null + + @Option(names = ["--specs-dir"], description = ["Directory with the API specification files"], required = false) + val specsDir: File? = null + + @Option( + names = ["--examples-base-dir"], + description = ["Base directory which contains multiple external examples directories each named as per the Specmatic naming convention to associate them with the corresponding API specification"], + required = false + ) + val examplesBaseDir: File? = null + @Option(names = ["--debug"], description = ["Debug logs"]) var verbose = false @@ -198,81 +216,167 @@ For example: var filterNotName: String = "" override fun call(): Int { + if (contractFile != null && exampleFile != null) return validateExampleFile(contractFile!!, exampleFile) + + if (contractFile != null && examplesDir != null) { + val (exitCode, validationResults) = validateExamplesDir(contractFile!!, examplesDir) + + printValidationResult(validationResults, "Example directory") + if (exitCode == 1) return FAILURE_EXIT_CODE + if (validationResults.containsFailure()) return FAILURE_EXIT_CODE + return SUCCESS_EXIT_CODE + } + + if (contractFile != null) return validateImplicitExamplesFrom(contractFile!!) + + if (specsDir != null && examplesBaseDir != null) { + val exitCode = validateAllExamplesAssociatedToEachSpecIn(specsDir, examplesBaseDir) + return exitCode + } + if (specsDir != null) { + val exitCode = validateAllExamplesAssociatedToEachSpecIn(specsDir, specsDir) + return exitCode + } + + logger.log("Invalid combination of CLI options. Please refer to the help section using --help command to understand how to use this command") + return FAILURE_EXIT_CODE + } + + private fun validateExampleFile(contractFile: File, exampleFile: File): Int { if (!contractFile.exists()) { logger.log("Could not find file ${contractFile.path}") - return 1 + return FAILURE_EXIT_CODE } configureLogger(this.verbose) - if (exampleFile != null) { - try { - validateSingleExample(contractFile, exampleFile).throwOnFailure() + try { + validateSingleExample(contractFile, exampleFile).throwOnFailure() - logger.log("The provided example ${exampleFile.name} is valid.") - } catch (e: ContractException) { - logger.log("The provided example ${exampleFile.name} is invalid. Reason:\n") - logger.log(exceptionCauseMessage(e)) - return 1 - } - } else { - val scenarioFilter = ExamplesInteractiveServer.ScenarioFilter(filterName, filterNotName, filter, filterNot) + logger.log("The provided example ${exampleFile.name} is valid.") + return SUCCESS_EXIT_CODE + } catch (e: ContractException) { + logger.log("The provided example ${exampleFile.name} is invalid. Reason:\n") + logger.log(exceptionCauseMessage(e)) + return FAILURE_EXIT_CODE + } + } + + private fun validateExamplesDir(contractFile: File, examplesDir: File, enableLogging: Boolean = true): Pair> { + val feature = parseContractFileToFeature(contractFile) + val (externalExampleDir, externalExamples) = loadExternalExamples(examplesDir = examplesDir) + if (!externalExampleDir.exists()) { + logger.log("$externalExampleDir does not exist, did not find any files to validate") + return FAILURE_EXIT_CODE to emptyMap() + } + if (externalExamples.none()) { + logger.log("No example files found in $externalExampleDir") + return FAILURE_EXIT_CODE to emptyMap() + } + return SUCCESS_EXIT_CODE to validateExternalExamples(feature, externalExamples, enableLogging) + } - val (validateInline, validateExternal) = if(!Flags.getBooleanValue("VALIDATE_INLINE_EXAMPLES") && !Flags.getBooleanValue("IGNORE_INLINE_EXAMPLES")) { - true to true - } else { - Flags.getBooleanValue("VALIDATE_INLINE_EXAMPLES") to Flags.getBooleanValue("IGNORE_INLINE_EXAMPLES") + private fun validateAllExamplesAssociatedToEachSpecIn( + specsDir: File, + examplesBaseDir: File + ): Int { + val validationResults = specsDir.walk().filter { it.isFile }.flatMapIndexed { index, it -> + val associatedExamplesDir = examplesBaseDir.associatedExampleDirFor(it) ?: return@flatMapIndexed emptyList() + + logger.log("${index.inc()}. Validating examples in ${associatedExamplesDir.name} associated to ${it.name}...${System.lineSeparator()}") + val results = validateExamplesDir(it, associatedExamplesDir, false).second.entries.map { entry -> + entry.toPair() } - val feature = parseContractFileToFeature(contractFile) + printValidationResult(results.toMap(), "The ${associatedExamplesDir.name} Directory") + logger.log(System.lineSeparator()) + results + }.toMap() + logger.log("Summary:") + printValidationResult(validationResults, "Overall") + if (validationResults.containsFailure()) return FAILURE_EXIT_CODE + return SUCCESS_EXIT_CODE + } - val inlineExampleValidationResults = if(validateInline) { - val inlineExamples = feature.stubsFromExamples.mapValues { - it.value.map { - ScenarioStub(it.first, it.second) - } - } + private fun validateImplicitExamplesFrom(contractFile: File): Int { + val feature = parseContractFileToFeature(contractFile) - ExamplesInteractiveServer.validateMultipleExamples(feature, examples = inlineExamples, inline = true, scenarioFilter = scenarioFilter) - } else emptyMap() + val (validateInline, validateExternal) = getValidateInlineAndValidateExternalFlags() - val externalExampleValidationResults = if(validateExternal) { - val (externalExampleDir, externalExamples) = loadExternalExamples(contractFile) + val inlineExampleValidationResults = if (!validateInline) emptyMap() + else validateInlineExamples(feature) - if(!externalExampleDir.exists()) { - logger.log("$externalExampleDir does not exist, did not find any files to validate") - return 1 - } + val externalExampleValidationResults = if (!validateExternal) emptyMap() + else { + val (exitCode, validationResults) + = validateExamplesDir(contractFile, defaultExternalExampleDirFrom(contractFile)) + if(exitCode == 1) exitProcess(1) + validationResults + } - if(externalExamples.none()) { - logger.log("No example files found in $externalExampleDir") - return 1 - } + val hasFailures = + inlineExampleValidationResults.containsFailure() || externalExampleValidationResults.containsFailure() + + printValidationResult(inlineExampleValidationResults, "Inline example") + printValidationResult(externalExampleValidationResults, "Example file") + + if (hasFailures) return FAILURE_EXIT_CODE + return SUCCESS_EXIT_CODE + } - ExamplesInteractiveServer.validateMultipleExamples(feature, examples = externalExamples, scenarioFilter = scenarioFilter) - } else emptyMap() + private fun validateInlineExamples(feature: Feature): Map { + return ExamplesInteractiveServer.validateExamples( + feature, + examples = feature.stubsFromExamples.mapValues { (_, stub) -> + stub.map { (request, response) -> + ScenarioStub(request, response) + } + }, + inline = true, + scenarioFilter = ExamplesInteractiveServer.ScenarioFilter( + filterName, + filterNotName, + filter, + filterNot + ) + ) + } - val hasFailures = inlineExampleValidationResults.any { it.value is Result.Failure } || externalExampleValidationResults.any { it.value is Result.Failure } + private fun validateExternalExamples( + feature: Feature, + externalExamples: Map>, + enableLogging: Boolean = true + ): Map { + return ExamplesInteractiveServer.validateExamples( + feature, + examples = externalExamples, + scenarioFilter = ExamplesInteractiveServer.ScenarioFilter( + filterName, + filterNotName, + filter, + filterNot + ), + enableLogging = enableLogging + ) + } - printValidationResult(inlineExampleValidationResults, "Inline example") - printValidationResult(externalExampleValidationResults, "Example file") + private fun getValidateInlineAndValidateExternalFlags(): Pair { + return when { + !Flags.getBooleanValue("VALIDATE_INLINE_EXAMPLES") && !Flags.getBooleanValue( + "IGNORE_INLINE_EXAMPLES" + ) -> true to true - if(hasFailures) - return 1 + else -> Flags.getBooleanValue("VALIDATE_INLINE_EXAMPLES") to Flags.getBooleanValue("IGNORE_INLINE_EXAMPLES") } - - return 0 } private fun printValidationResult(validationResults: Map, tag: String) { - if(validationResults.isEmpty()) + if (validationResults.isEmpty()) return - val hasFailures = validationResults.any { it.value is Result.Failure } + val titleTag = tag.split(" ").joinToString(" ") { if (it.isBlank()) it else it.capitalizeFirstChar() } - val titleTag = tag.split(" ").joinToString(" ") { if(it.isBlank()) it else it.capitalizeFirstChar() } - - if(hasFailures) { + if (validationResults.containsFailure()) { println() logger.log("=============== $titleTag Validation Results ===============") @@ -290,6 +394,16 @@ For example: logger.log(Results(validationResults.values.toList()).summary()) logger.log("=".repeat(summaryTitle.length)) } + + private fun Map.containsFailure(): Boolean { + return this.any { it.value is Result.Failure } + } + + private fun File.associatedExampleDirFor(specFile: File): File? { + return this.walk().firstOrNull { exampleDir -> + exampleDir.isFile.not() && exampleDir.nameWithoutExtension == "${specFile.nameWithoutExtension}_examples" + } + } } @Command( diff --git a/core/src/main/kotlin/io/specmatic/core/examples/server/ExamplesInteractiveServer.kt b/core/src/main/kotlin/io/specmatic/core/examples/server/ExamplesInteractiveServer.kt index 1240ccb5c..d8bdbfb98 100644 --- a/core/src/main/kotlin/io/specmatic/core/examples/server/ExamplesInteractiveServer.kt +++ b/core/src/main/kotlin/io/specmatic/core/examples/server/ExamplesInteractiveServer.kt @@ -151,7 +151,7 @@ class ExamplesInteractiveServer( exampleFilePath to listOf(ScenarioStub.readFromFile(File(exampleFilePath))) } - val results = validateMultipleExamples(contractFile, examples = examples) + val results = validateExamples(contractFile, examples = examples) val validationResults = results.map { (exampleFilePath, result) -> try { @@ -510,16 +510,22 @@ class ExamplesInteractiveServer( } } - fun validateMultipleExamples(contractFile: File, examples: Map> = emptyMap(), scenarioFilter: ScenarioFilter = ScenarioFilter()): Map { + fun validateExamples(contractFile: File, examples: Map> = emptyMap(), scenarioFilter: ScenarioFilter = ScenarioFilter()): Map { val feature = parseContractFileToFeature(contractFile) - return validateMultipleExamples(feature, examples, false, scenarioFilter) + return validateExamples(feature, examples, false, scenarioFilter) } - fun validateMultipleExamples(feature: Feature, examples: Map> = emptyMap(), inline: Boolean = false, scenarioFilter: ScenarioFilter = ScenarioFilter()): Map { + fun validateExamples( + feature: Feature, + examples: Map> = emptyMap(), + inline: Boolean = false, + scenarioFilter: ScenarioFilter = ScenarioFilter(), + enableLogging: Boolean = true + ): Map { val updatedFeature = scenarioFilter.filter(feature) val results = examples.mapValues { (name, exampleList) -> - logger.log("Validating $name") + if(enableLogging) logger.log("Validating $name") exampleList.mapNotNull { example -> try { @@ -717,20 +723,22 @@ data class ExampleTestResponse( } } -fun loadExternalExamples(contractFile: File): Pair>> { - val examplesDir = - contractFile.absoluteFile.parentFile.resolve(contractFile.nameWithoutExtension + "_examples") +fun loadExternalExamples( + examplesDir: File +): Pair>> { if (!examplesDir.isDirectory) { logger.log("$examplesDir does not exist, did not find any files to validate") exitProcess(1) } return examplesDir to examplesDir.walk().mapNotNull { - if (it.isFile) - Pair(it.path, it) - else - null + if (it.isFile.not()) return@mapNotNull null + Pair(it.path, it) }.toMap().mapValues { listOf(ScenarioStub.readFromFile(it.value)) } } + +fun defaultExternalExampleDirFrom(contractFile: File): File { + return contractFile.absoluteFile.parentFile.resolve(contractFile.nameWithoutExtension + "_examples") +}