Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enhance example validate command to support validation of a specs directory #1389

Merged
merged 4 commits into from
Oct 27, 2024
Merged

Enhance example validate command to support validation of a specs directory #1389

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1055744
Enhance examples validate command to accept --examples-dir & --spec-dir
yogeshnikam671 Oct 26, 2024
9ed5a3b
Introduce --examples-dir instead of reusing the same --example-dir fo…
yogeshnikam671 Oct 26, 2024
8993d95
Add a variant in validate command where the specs-dir contains both t…
yogeshnikam671 Oct 26, 2024
cf4201a
Update descriptions of certain CLI arguments and refactor some stuff
yogeshnikam671 Oct 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
226 changes: 170 additions & 56 deletions application/src/main/kotlin/application/ExamplesCommand.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
package application

import io.specmatic.core.Feature
import io.specmatic.core.Result
import io.specmatic.core.Results
import io.specmatic.core.SPECMATIC_STUB_DICTIONARY
import io.specmatic.core.examples.server.ExamplesInteractiveServer
import io.specmatic.core.examples.server.ExamplesInteractiveServer.Companion.validateSingleExample
import io.specmatic.core.examples.server.defaultExternalExampleDirFrom
import io.specmatic.core.examples.server.loadExternalExamples
import io.specmatic.core.filters.ScenarioMetadataFilter
import io.specmatic.core.log.*
import io.specmatic.core.parseContractFileToFeature
import io.specmatic.core.pattern.ContractException
Expand All @@ -16,6 +17,10 @@ import picocli.CommandLine.*
import java.io.File
import java.lang.Thread.sleep
import java.util.concurrent.Callable
import kotlin.system.exitProcess

private const val SUCCESS_EXIT_CODE = 0
private const val FAILURE_EXIT_CODE = 1

@Command(
name = "examples",
Expand Down Expand Up @@ -99,11 +104,11 @@ For example:
override fun call(): Int {
if (contractFile == null) {
println("No contract file provided. Use a subcommand or provide a contract file. Use --help for more details.")
return 1
return FAILURE_EXIT_CODE
}
if (!contractFile!!.exists()) {
logger.log("Could not find file ${contractFile!!.path}")
return 1
return FAILURE_EXIT_CODE
}

configureLogger(this.verbose)
Expand All @@ -120,10 +125,10 @@ For example:
)
} catch (e: Throwable) {
logger.log(e)
return 1
return FAILURE_EXIT_CODE
}

return 0
return SUCCESS_EXIT_CODE
}

@Command(
Expand Down Expand Up @@ -174,12 +179,25 @@ For example:
)
var filterNot: List<String> = emptyList()

@Option(names = ["--contract-file"], description = ["Contract file path"], required = true)
lateinit var contractFile: File
@Option(names = ["--contract-file", "--spec-file"], description = ["Contract file path"], required = false)
var contractFile: File? = null

@Option(names = ["--example-file"], description = ["Example file path"], required = false)
val exampleFile: File? = null

@Option(names = ["--examples-dir"], description = ["External examples directory path for a single API specification (If you are not following the default naming convention for external examples directory)"], required = false)
val examplesDir: File? = null

@Option(names = ["--specs-dir"], description = ["Directory with the API specification files"], required = false)
val specsDir: File? = null

@Option(
names = ["--examples-base-dir"],
description = ["Base directory which contains multiple external examples directories each named as per the Specmatic naming convention to associate them with the corresponding API specification"],
required = false
)
val examplesBaseDir: File? = null

@Option(names = ["--debug"], description = ["Debug logs"])
var verbose = false

Expand All @@ -198,81 +216,167 @@ For example:
var filterNotName: String = ""

override fun call(): Int {
if (contractFile != null && exampleFile != null) return validateExampleFile(contractFile!!, exampleFile)

if (contractFile != null && examplesDir != null) {
val (exitCode, validationResults) = validateExamplesDir(contractFile!!, examplesDir)

printValidationResult(validationResults, "Example directory")
if (exitCode == 1) return FAILURE_EXIT_CODE
if (validationResults.containsFailure()) return FAILURE_EXIT_CODE
return SUCCESS_EXIT_CODE
}

if (contractFile != null) return validateImplicitExamplesFrom(contractFile!!)

if (specsDir != null && examplesBaseDir != null) {
val exitCode = validateAllExamplesAssociatedToEachSpecIn(specsDir, examplesBaseDir)
return exitCode
}
if (specsDir != null) {
val exitCode = validateAllExamplesAssociatedToEachSpecIn(specsDir, specsDir)
return exitCode
}

logger.log("Invalid combination of CLI options. Please refer to the help section using --help command to understand how to use this command")
return FAILURE_EXIT_CODE
}

private fun validateExampleFile(contractFile: File, exampleFile: File): Int {
if (!contractFile.exists()) {
logger.log("Could not find file ${contractFile.path}")
return 1
return FAILURE_EXIT_CODE
}

configureLogger(this.verbose)

if (exampleFile != null) {
try {
validateSingleExample(contractFile, exampleFile).throwOnFailure()
try {
validateSingleExample(contractFile, exampleFile).throwOnFailure()

logger.log("The provided example ${exampleFile.name} is valid.")
} catch (e: ContractException) {
logger.log("The provided example ${exampleFile.name} is invalid. Reason:\n")
logger.log(exceptionCauseMessage(e))
return 1
}
} else {
val scenarioFilter = ExamplesInteractiveServer.ScenarioFilter(filterName, filterNotName, filter, filterNot)
logger.log("The provided example ${exampleFile.name} is valid.")
return SUCCESS_EXIT_CODE
} catch (e: ContractException) {
logger.log("The provided example ${exampleFile.name} is invalid. Reason:\n")
logger.log(exceptionCauseMessage(e))
return FAILURE_EXIT_CODE
}
}

private fun validateExamplesDir(contractFile: File, examplesDir: File, enableLogging: Boolean = true): Pair<Int, Map<String, Result>> {
val feature = parseContractFileToFeature(contractFile)
val (externalExampleDir, externalExamples) = loadExternalExamples(examplesDir = examplesDir)
if (!externalExampleDir.exists()) {
logger.log("$externalExampleDir does not exist, did not find any files to validate")
return FAILURE_EXIT_CODE to emptyMap()
}
if (externalExamples.none()) {
logger.log("No example files found in $externalExampleDir")
return FAILURE_EXIT_CODE to emptyMap()
}
return SUCCESS_EXIT_CODE to validateExternalExamples(feature, externalExamples, enableLogging)
}

val (validateInline, validateExternal) = if(!Flags.getBooleanValue("VALIDATE_INLINE_EXAMPLES") && !Flags.getBooleanValue("IGNORE_INLINE_EXAMPLES")) {
true to true
} else {
Flags.getBooleanValue("VALIDATE_INLINE_EXAMPLES") to Flags.getBooleanValue("IGNORE_INLINE_EXAMPLES")
private fun validateAllExamplesAssociatedToEachSpecIn(
specsDir: File,
examplesBaseDir: File
): Int {
val validationResults = specsDir.walk().filter { it.isFile }.flatMapIndexed { index, it ->
val associatedExamplesDir = examplesBaseDir.associatedExampleDirFor(it) ?: return@flatMapIndexed emptyList()

logger.log("${index.inc()}. Validating examples in ${associatedExamplesDir.name} associated to ${it.name}...${System.lineSeparator()}")
val results = validateExamplesDir(it, associatedExamplesDir, false).second.entries.map { entry ->
entry.toPair()
}

val feature = parseContractFileToFeature(contractFile)
printValidationResult(results.toMap(), "The ${associatedExamplesDir.name} Directory")
logger.log(System.lineSeparator())
results
}.toMap()
logger.log("Summary:")
printValidationResult(validationResults, "Overall")
if (validationResults.containsFailure()) return FAILURE_EXIT_CODE
return SUCCESS_EXIT_CODE
}

val inlineExampleValidationResults = if(validateInline) {
val inlineExamples = feature.stubsFromExamples.mapValues {
it.value.map {
ScenarioStub(it.first, it.second)
}
}
private fun validateImplicitExamplesFrom(contractFile: File): Int {
val feature = parseContractFileToFeature(contractFile)

ExamplesInteractiveServer.validateMultipleExamples(feature, examples = inlineExamples, inline = true, scenarioFilter = scenarioFilter)
} else emptyMap()
val (validateInline, validateExternal) = getValidateInlineAndValidateExternalFlags()

val externalExampleValidationResults = if(validateExternal) {
val (externalExampleDir, externalExamples) = loadExternalExamples(contractFile)
val inlineExampleValidationResults = if (!validateInline) emptyMap()
else validateInlineExamples(feature)

if(!externalExampleDir.exists()) {
logger.log("$externalExampleDir does not exist, did not find any files to validate")
return 1
}
val externalExampleValidationResults = if (!validateExternal) emptyMap()
else {
val (exitCode, validationResults)
= validateExamplesDir(contractFile, defaultExternalExampleDirFrom(contractFile))
if(exitCode == 1) exitProcess(1)
validationResults
}

if(externalExamples.none()) {
logger.log("No example files found in $externalExampleDir")
return 1
}
val hasFailures =
inlineExampleValidationResults.containsFailure() || externalExampleValidationResults.containsFailure()

printValidationResult(inlineExampleValidationResults, "Inline example")
printValidationResult(externalExampleValidationResults, "Example file")

if (hasFailures) return FAILURE_EXIT_CODE
return SUCCESS_EXIT_CODE
}

ExamplesInteractiveServer.validateMultipleExamples(feature, examples = externalExamples, scenarioFilter = scenarioFilter)
} else emptyMap()
private fun validateInlineExamples(feature: Feature): Map<String, Result> {
return ExamplesInteractiveServer.validateExamples(
feature,
examples = feature.stubsFromExamples.mapValues { (_, stub) ->
stub.map { (request, response) ->
ScenarioStub(request, response)
}
},
inline = true,
harikrishnan83 marked this conversation as resolved.
Show resolved Hide resolved
scenarioFilter = ExamplesInteractiveServer.ScenarioFilter(
filterName,
filterNotName,
filter,
filterNot
)
)
}

val hasFailures = inlineExampleValidationResults.any { it.value is Result.Failure } || externalExampleValidationResults.any { it.value is Result.Failure }
private fun validateExternalExamples(
feature: Feature,
externalExamples: Map<String, List<ScenarioStub>>,
enableLogging: Boolean = true
): Map<String, Result> {
return ExamplesInteractiveServer.validateExamples(
feature,
examples = externalExamples,
scenarioFilter = ExamplesInteractiveServer.ScenarioFilter(
filterName,
filterNotName,
filter,
filterNot
),
enableLogging = enableLogging
)
}

printValidationResult(inlineExampleValidationResults, "Inline example")
printValidationResult(externalExampleValidationResults, "Example file")
private fun getValidateInlineAndValidateExternalFlags(): Pair<Boolean, Boolean> {
return when {
!Flags.getBooleanValue("VALIDATE_INLINE_EXAMPLES") && !Flags.getBooleanValue(
"IGNORE_INLINE_EXAMPLES"
) -> true to true

if(hasFailures)
return 1
else -> Flags.getBooleanValue("VALIDATE_INLINE_EXAMPLES") to Flags.getBooleanValue("IGNORE_INLINE_EXAMPLES")
}

return 0
}

private fun printValidationResult(validationResults: Map<String, Result>, tag: String) {
if(validationResults.isEmpty())
if (validationResults.isEmpty())
return

val hasFailures = validationResults.any { it.value is Result.Failure }
val titleTag = tag.split(" ").joinToString(" ") { if (it.isBlank()) it else it.capitalizeFirstChar() }

val titleTag = tag.split(" ").joinToString(" ") { if(it.isBlank()) it else it.capitalizeFirstChar() }

if(hasFailures) {
if (validationResults.containsFailure()) {
println()
logger.log("=============== $titleTag Validation Results ===============")

Expand All @@ -290,6 +394,16 @@ For example:
logger.log(Results(validationResults.values.toList()).summary())
logger.log("=".repeat(summaryTitle.length))
}

private fun Map<String, Result>.containsFailure(): Boolean {
return this.any { it.value is Result.Failure }
}

private fun File.associatedExampleDirFor(specFile: File): File? {
return this.walk().firstOrNull { exampleDir ->
exampleDir.isFile.not() && exampleDir.nameWithoutExtension == "${specFile.nameWithoutExtension}_examples"
}
}
}

@Command(
Expand Down
32 changes: 20 additions & 12 deletions core/src/main/kotlin/io/specmatic/core/examples/server/ExamplesInteractiveServer.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ class ExamplesInteractiveServer(
exampleFilePath to listOf(ScenarioStub.readFromFile(File(exampleFilePath)))
}

val results = validateMultipleExamples(contractFile, examples = examples)
val results = validateExamples(contractFile, examples = examples)

val validationResults = results.map { (exampleFilePath, result) ->
try {
Expand Down Expand Up @@ -510,16 +510,22 @@ class ExamplesInteractiveServer(
}
}

fun validateMultipleExamples(contractFile: File, examples: Map<String, List<ScenarioStub>> = emptyMap(), scenarioFilter: ScenarioFilter = ScenarioFilter()): Map<String, Result> {
fun validateExamples(contractFile: File, examples: Map<String, List<ScenarioStub>> = emptyMap(), scenarioFilter: ScenarioFilter = ScenarioFilter()): Map<String, Result> {
val feature = parseContractFileToFeature(contractFile)
return validateMultipleExamples(feature, examples, false, scenarioFilter)
return validateExamples(feature, examples, false, scenarioFilter)
}

fun validateMultipleExamples(feature: Feature, examples: Map<String, List<ScenarioStub>> = emptyMap(), inline: Boolean = false, scenarioFilter: ScenarioFilter = ScenarioFilter()): Map<String, Result> {
fun validateExamples(
feature: Feature,
examples: Map<String, List<ScenarioStub>> = emptyMap(),
inline: Boolean = false,
scenarioFilter: ScenarioFilter = ScenarioFilter(),
enableLogging: Boolean = true
): Map<String, Result> {
val updatedFeature = scenarioFilter.filter(feature)

val results = examples.mapValues { (name, exampleList) ->
logger.log("Validating $name")
if(enableLogging) logger.log("Validating $name")

exampleList.mapNotNull { example ->
try {
Expand Down Expand Up @@ -717,20 +723,22 @@ data class ExampleTestResponse(
}
}

fun loadExternalExamples(contractFile: File): Pair<File, Map<String, List<ScenarioStub>>> {
val examplesDir =
contractFile.absoluteFile.parentFile.resolve(contractFile.nameWithoutExtension + "_examples")
fun loadExternalExamples(
examplesDir: File
): Pair<File, Map<String, List<ScenarioStub>>> {
if (!examplesDir.isDirectory) {
logger.log("$examplesDir does not exist, did not find any files to validate")
exitProcess(1)
}

return examplesDir to examplesDir.walk().mapNotNull {
if (it.isFile)
Pair(it.path, it)
else
null
if (it.isFile.not()) return@mapNotNull null
Pair(it.path, it)
}.toMap().mapValues {
listOf(ScenarioStub.readFromFile(it.value))
}
}

fun defaultExternalExampleDirFrom(contractFile: File): File {
return contractFile.absoluteFile.parentFile.resolve(contractFile.nameWithoutExtension + "_examples")
}