Add --strict flag in TestCommand. If this flag is true, tests should not run if example loading fails

application/src/main/kotlin/application/TestCommand.kt

@@ -25,6 +25,7 @@ import io.specmatic.test.SpecmaticJUnitSupport.Companion.HOST
 import io.specmatic.test.SpecmaticJUnitSupport.Companion.INLINE_SUGGESTIONS
 import io.specmatic.test.SpecmaticJUnitSupport.Companion.OVERLAY_FILE_PATH
 import io.specmatic.test.SpecmaticJUnitSupport.Companion.PORT
+import io.specmatic.test.SpecmaticJUnitSupport.Companion.STRICT_MODE
 import io.specmatic.test.SpecmaticJUnitSupport.Companion.SUGGESTIONS_PATH
 import io.specmatic.test.SpecmaticJUnitSupport.Companion.TEST_BASE_URL
 import io.specmatic.test.SpecmaticJUnitSupport.Companion.VARIABLES_FILE_NAME
@@ -157,6 +158,13 @@ For example:
     @Option(names = ["--overlay-file"], description = ["Overlay file for the specification"], required = false)
     var overlayFilePath: String? = null
 
+    @Option(
+        names = ["--strict"],
+        description = ["If true, tests will only run if all the examples are valid"],
+        required = false
+    )
+    var strictMode: Boolean = false
+
     override fun call() = try {
         setParallelism()
 
@@ -194,6 +202,7 @@ For example:
         System.setProperty(FILTER, filter.joinToString(";"))
         System.setProperty(FILTER_NOT, filterNot.joinToString(";"))
         System.setProperty(OVERLAY_FILE_PATH, overlayFilePath.orEmpty())
+        System.setProperty(STRICT_MODE, strictMode.toString())
 
         if(exampleDirs.isNotEmpty()) {
             System.setProperty(EXAMPLE_DIRECTORIES, exampleDirs.joinToString(","))

core/src/main/kotlin/io/specmatic/conversions/OpenApiSpecification.kt

@@ -72,7 +72,8 @@ class OpenApiSpecification(
     private val specificationPath: String? = null,
     private val securityConfiguration: SecurityConfiguration? = null,
     private val specmaticConfig: SpecmaticConfig = SpecmaticConfig(),
-    private val dictionary: Map<String, Value> = loadDictionary(openApiFilePath, specmaticConfig.stub.dictionary)
+    private val dictionary: Map<String, Value> = loadDictionary(openApiFilePath, specmaticConfig.stub.dictionary),
+    private val strictMode: Boolean = false
 ) : IncludedSpecification, ApiSpecification {
     init {
         logger.log(openApiSpecificationInfo(openApiFilePath, parsedOpenApi))
@@ -130,7 +131,8 @@ class OpenApiSpecification(
             specificationPath: String? = null,
             securityConfiguration: SecurityConfiguration? = null,
             specmaticConfig: SpecmaticConfig = SpecmaticConfig(),
-            overlayContent: String = ""
+            overlayContent: String = "",
+            strictMode: Boolean = false
         ): OpenApiSpecification {
             val implicitOverlayFile = getImplicitOverlayContent(openApiFilePath)
 
@@ -164,6 +166,7 @@ class OpenApiSpecification(
                 specificationPath,
                 securityConfiguration,
                 specmaticConfig,
+                strictMode = strictMode
             )
         }
 
@@ -249,6 +252,7 @@ class OpenApiSpecification(
             serviceType = SERVICE_TYPE_HTTP,
             stubsFromExamples = stubsFromExamples,
             specmaticConfig = specmaticConfig,
+            strictMode = strictMode
         )
     }
 

core/src/main/kotlin/io/specmatic/core/Feature.kt

@@ -38,7 +38,8 @@ fun parseContractFileToFeature(
     specificationPath: String? = null,
     securityConfiguration: SecurityConfiguration? = null,
     specmaticConfig: SpecmaticConfig = loadSpecmaticConfigOrDefault(getConfigFilePath()),
-    overlayContent: String = ""
+    overlayContent: String = "",
+    strictMode: Boolean = false
 ): Feature {
     return parseContractFileToFeature(
         File(contractPath),
@@ -49,7 +50,8 @@ fun parseContractFileToFeature(
         specificationPath,
         securityConfiguration,
         specmaticConfig,
-        overlayContent
+        overlayContent,
+        strictMode
     )
 }
 
@@ -67,7 +69,8 @@ fun parseContractFileToFeature(
     specificationPath: String? = null,
     securityConfiguration: SecurityConfiguration? = null,
     specmaticConfig: SpecmaticConfig = loadSpecmaticConfigOrDefault(getConfigFilePath()),
-    overlayContent: String = ""
+    overlayContent: String = "",
+    strictMode: Boolean = false
 ): Feature {
     logger.debug("Parsing contract file ${file.path}, absolute path ${file.absolutePath}")
     return when (file.extension) {
@@ -80,7 +83,8 @@ fun parseContractFileToFeature(
             specificationPath = specificationPath,
             securityConfiguration = securityConfiguration,
             specmaticConfig = specmaticConfig,
-            overlayContent = overlayContent
+            overlayContent = overlayContent,
+            strictMode = strictMode
         ).toFeature()
         WSDL -> wsdlContentToFeature(checkExists(file).readText(), file.canonicalPath)
         in CONTRACT_EXTENSIONS -> parseGherkinStringToFeature(checkExists(file).readText().trim(), file.canonicalPath)
@@ -120,7 +124,8 @@ data class Feature(
     val serviceType:String? = null,
     val stubsFromExamples: Map<String, List<Pair<HttpRequest, HttpResponse>>> = emptyMap(),
     val specmaticConfig: SpecmaticConfig = SpecmaticConfig(),
-    val flagsBased: FlagsBased = strategiesFromFlags(specmaticConfig)
+    val flagsBased: FlagsBased = strategiesFromFlags(specmaticConfig),
+    val strictMode: Boolean = false
 ): IFeature {
     fun enableGenerativeTesting(onlyPositive: Boolean = false): Feature {
         val updatedSpecmaticConfig = specmaticConfig.copy(
@@ -1727,7 +1732,9 @@ data class Feature(
                     ) to exampleFromFile.toRow(specmaticConfig)
                 }
             } catch (e: Throwable) {
-                logger.log(e, "Error reading file ${exampleFromFile.expectationFilePath}")
+                val errorMessage = "Error reading file ${exampleFromFile.expectationFilePath}"
+                if(strictMode) throw ContractException(errorMessage)
+                logger.log(e, errorMessage)
                 null
             }
         }

junit5-support/src/main/kotlin/io/specmatic/test/SpecmaticJUnitSupport.kt

@@ -70,6 +70,7 @@ open class SpecmaticJUnitSupport {
         const val FILTER_NAME_ENVIRONMENT_VARIABLE = "FILTER_NAME"
         const val FILTER_NOT_NAME_ENVIRONMENT_VARIABLE = "FILTER_NOT_NAME"
         const val OVERLAY_FILE_PATH = "overlayFilePath"
+        const val STRICT_MODE = "strictMode"
         private const val ENDPOINTS_API = "endpointsAPI"
 
         val partialSuccesses: MutableList<Result.Success> = mutableListOf()
@@ -463,6 +464,7 @@ open class SpecmaticJUnitSupport {
             return Pair(emptySequence(), emptyList())
 
         val contractFile = File(path)
+        val strictMode = (System.getProperty(STRICT_MODE) ?: System.getenv(STRICT_MODE)) == "true"
         val feature =
             parseContractFileToFeature(
                 contractFile.path,
@@ -473,7 +475,8 @@ open class SpecmaticJUnitSupport {
                 specificationPath,
                 securityConfiguration,
                 specmaticConfig = specmaticConfig ?: SpecmaticConfig(),
-                overlayContent = overlayContent
+                overlayContent = overlayContent,
+                strictMode = strictMode
             ).copy(testVariables = config.variables, testBaseURLs = config.baseURLs).loadExternalisedExamples()
 
         feature.validateExamplesOrException()