Merge pull request #618 from znsio/invalid_request_examples

Invalid request examples
znsio · Mar 8, 2023 · 4dcf51a · 4dcf51a
2 parents 92d97c2 + 540e1d8
commit 4dcf51a
Show file tree

Hide file tree

Showing 4 changed files with 259 additions and 19 deletions.
diff --git a/core/src/main/kotlin/in/specmatic/core/HttpRequestPattern.kt b/core/src/main/kotlin/in/specmatic/core/HttpRequestPattern.kt
@@ -367,7 +367,12 @@ data class HttpRequestPattern(
         }
     }
 
-    fun newBasedOn(row: Row, resolver: Resolver, status: Int = 0): List<HttpRequestPattern> {
+    fun newBasedOn(row: Row, initialResolver: Resolver, status: Int = 0): List<HttpRequestPattern> {
+        val resolver = if(status in invalidRequestStatuses)
+            initialResolver.invalidRequestResolver()
+        else
+            initialResolver
+
         return attempt(breadCrumb = "REQUEST") {
             val newURLMatchers = urlMatcher?.newBasedOn(row, resolver) ?: listOf<URLMatcher?>(null)
                 val newBodies: List<Pattern> = attempt(breadCrumb = "BODY") {

diff --git a/core/src/main/kotlin/in/specmatic/core/Resolver.kt b/core/src/main/kotlin/in/specmatic/core/Resolver.kt
@@ -5,6 +5,22 @@ import `in`.specmatic.core.value.StringValue
 import `in`.specmatic.core.value.True
 import `in`.specmatic.core.value.Value
 
+val actualMatch: (resolver: Resolver, factKey: String?, pattern: Pattern, sampleValue: Value) -> Result = { resolver: Resolver, factKey: String?, pattern: Pattern, sampleValue: Value ->
+    resolver.actualPatternMatch(factKey, pattern, sampleValue)
+}
+
+val matchAnything: (resolver: Resolver, factKey: String?, pattern: Pattern, sampleValue: Value) -> Result = { resolver: Resolver, factKey: String?, pattern: Pattern, sampleValue: Value ->
+    Result.Success()
+}
+
+val actualParse: (resolver: Resolver, pattern: Pattern, rowValue: String) -> Value = { resolver: Resolver, pattern: Pattern, rowValue: String ->
+    resolver.actualParse(pattern, rowValue)
+}
+
+val alwaysReturnStringValue: (resolver: Resolver, pattern: Pattern, rowValue: String) -> Value = { resolver: Resolver, pattern: Pattern, rowValue: String ->
+    StringValue(rowValue)
+}
+
 data class Resolver(
     val factStore: FactStore = CheckFacts(),
     val mockMode: Boolean = false,
@@ -13,6 +29,8 @@ data class Resolver(
     val context: Map<String, String> = emptyMap(),
     val mismatchMessages: MismatchMessages = DefaultMismatchMessages,
     val isNegative: Boolean = false,
+    val patternMatchStrategy: (resolver: Resolver, factKey: String?, pattern: Pattern, sampleValue: Value) -> Result = actualMatch,
+    val parseStrategy: (resolver: Resolver, pattern: Pattern, rowValue: String) -> Value = actualParse,
     val generativeTestingEnabled: Boolean = false,
     val cyclePreventionStack: List<Pattern> = listOf(),
 ) {
@@ -38,6 +56,10 @@ data class Resolver(
     }
 
     fun matchesPattern(factKey: String?, pattern: Pattern, sampleValue: Value): Result {
+        return patternMatchStrategy(this, factKey, pattern, sampleValue)
+    }
+
+    fun actualPatternMatch(factKey: String?, pattern: Pattern, sampleValue: Value): Result {
         if (mockMode
                 && sampleValue is StringValue
                 && isPatternToken(sampleValue.string)
@@ -123,6 +145,14 @@ data class Resolver(
     }
 
     fun parse(pattern: Pattern, rowValue: String): Value {
+        return parseStrategy(this, pattern, rowValue)
+    }
+
+    fun actualParse(pattern: Pattern, rowValue: String): Value {
         return pattern.parse(rowValue, this)
     }
+
+    fun invalidRequestResolver(): Resolver {
+        return this.copy(patternMatchStrategy = matchAnything, parseStrategy = alwaysReturnStringValue)
+    }
 }
diff --git a/core/src/main/kotlin/in/specmatic/core/pattern/TabularPattern.kt b/core/src/main/kotlin/in/specmatic/core/pattern/TabularPattern.kt
@@ -180,7 +180,7 @@ fun newBasedOn(row: Row, key: String, pattern: Pattern, resolver: Resolver): Lis
         row.containsField(keyWithoutOptionality) -> {
             val rowValue = row.getField(keyWithoutOptionality)
 
-            val fromExamples = if (isPatternToken(rowValue)) {
+            if (isPatternToken(rowValue)) {
                 val rowPattern = resolver.getPattern(rowValue)
 
                 attempt(breadCrumb = key) {
@@ -200,25 +200,11 @@ fun newBasedOn(row: Row, key: String, pattern: Pattern, resolver: Resolver): Lis
                     resolver.parse(pattern, rowValue)
                 }
 
-                when (val matchResult = pattern.matches(parsedRowValue, resolver)) {
+                when (val matchResult = resolver.matchesPattern(null, pattern, parsedRowValue)) {
                     is Result.Failure -> throw ContractException(matchResult.toFailureReport())
                     else -> listOf(ExactValuePattern(parsedRowValue))
                 }
             }
-
-            fromExamples
-
-//            if(Flags.negativeTestingEnabled()) {
-//                val vanilla = pattern.newBasedOn(Row(), resolver)
-//
-//                val remainder = vanilla.filterNot { vanillaType ->
-//                    fromExamples.any { item -> vanillaType.encompasses(item, resolver, resolver) is Result.Success }
-//                }
-//
-//                fromExamples.plus(remainder)
-//            } else {
-//                fromExamples
-//            }
         }
         else -> resolver.withCyclePrevention(pattern, isOptional(key)) { cyclePreventedResolver ->
             pattern.newBasedOn(row, cyclePreventedResolver)

diff --git a/core/src/test/kotlin/in/specmatic/conversions/OpenApiKtTest.kt b/core/src/test/kotlin/in/specmatic/conversions/OpenApiKtTest.kt
@@ -1996,7 +1996,7 @@ Feature: Foo API
     }
 
     @Test
-    fun `contract-invalid test should be allowed for 400 request`() {
+    fun `contract-invalid test should be allowed for 400 request payload`() {
         val contract = OpenApiSpecification.fromYAML("""
 openapi: "3.0.3"
 info:
@@ -2089,7 +2089,226 @@ components:
                     if(jsonBody.jsonObject["name"] is NumberValue)
                         contractInvalidValueReceived = true
 
-                    return HttpResponse(422, body = parsedJSONObject("""{"message": "invalid request"}"""))
+                    return HttpResponse(400, body = parsedJSONObject("""{"message": "invalid request"}"""))
+                }
+
+                override fun setServerState(serverState: Map<String, Value>) {
+                }
+            })
+
+            assertThat(contractInvalidValueReceived).isTrue
+        } finally {
+            System.clearProperty(Flags.negativeTestingFlag)
+        }
+    }
+
+    @Test
+    fun `contract-invalid test should be allowed for 400 query parameter`() {
+        val contract = OpenApiSpecification.fromYAML("""
+openapi: "3.0.3"
+info:
+  version: 1.0.0
+  title: Swagger Petstore
+  description: A sample API that uses a petstore as an example to demonstrate features in the OpenAPI 3.0 specification
+  termsOfService: http://swagger.io/terms/
+  contact:
+    name: Swagger API Team
+    email: apiteam@swagger.io
+    url: http://swagger.io
+  license:
+    name: Apache 2.0
+    url: https://www.apache.org/licenses/LICENSE-2.0.html
+servers:
+  - url: http://petstore.swagger.io/api
+paths:
+  /pets:
+    post:
+      summary: create a pet
+      description: Creates a new pet in the store. Duplicates are allowed
+      operationId: addPet
+      parameters:
+        - in: header
+          name: data
+          schema:
+            type: integer
+          examples:
+            INVALID:
+              value: hello
+            SUCCESS:
+              value: 10
+      requestBody:
+        description: Pet to add to the store
+        required: true
+        content:
+          application/json:
+            schema:
+              ${'$'}ref: '#/components/schemas/NewPet'
+            examples:
+              SUCCESS:
+                value:
+                  name: 'Archie'
+              INVALID:
+                value:
+                  name: 10
+      responses:
+        '200':
+          description: new pet record
+          content:
+            application/json:
+              schema:
+                ${'$'}ref: '#/components/schemas/Pet'
+              examples:
+                SUCCESS:
+                  value:
+                    id: 10
+                    name: Archie
+        '400':
+          description: invalid request
+          content:
+            application/json:
+              examples:
+                INVALID:
+                  value:
+                    message: Name must be a strings
+              schema:
+                type: object
+                properties:
+                  message:
+                    type: string
+components:
+  schemas:
+    Pet:
+      type: object
+      required:
+        - id
+        - name
+      properties:
+        name:
+          type: string
+        id:
+          type: integer
+    NewPet:
+      type: object
+      required:
+        - name
+      properties:
+        name:
+          type: string
+""".trimIndent(), "").toFeature()
+
+        var contractInvalidValueReceived = false
+
+        try {
+            contract.executeTests(object : TestExecutor {
+                override fun execute(request: HttpRequest): HttpResponse {
+                    val dataHeaderValue: String? = request.headers["data"]
+
+                    if(dataHeaderValue == "hello")
+                        contractInvalidValueReceived = true
+
+                    return HttpResponse(400, body = parsedJSONObject("""{"message": "invalid request"}"""))
+                }
+
+                override fun setServerState(serverState: Map<String, Value>) {
+                }
+            })
+
+            assertThat(contractInvalidValueReceived).isTrue
+        } finally {
+            System.clearProperty(Flags.negativeTestingFlag)
+        }
+    }
+
+    @Test
+    fun `contract-invalid test should be allowed for 400 request header`() {
+        val contract = OpenApiSpecification.fromYAML("""
+openapi: "3.0.3"
+info:
+  version: 1.0.0
+  title: Swagger Petstore
+  description: A sample API that uses a petstore as an example to demonstrate features in the OpenAPI 3.0 specification
+  termsOfService: http://swagger.io/terms/
+  contact:
+    name: Swagger API Team
+    email: apiteam@swagger.io
+    url: http://swagger.io
+  license:
+    name: Apache 2.0
+    url: https://www.apache.org/licenses/LICENSE-2.0.html
+servers:
+  - url: http://petstore.swagger.io/api
+paths:
+  /pets:
+    get:
+      summary: query for a pet
+      description: Queries info on a pet
+      parameters:
+        - in: query
+          name: data
+          schema:
+            type: integer
+          examples:
+            INVALID:
+              value: hello
+            SUCCESS:
+              value: 10
+      responses:
+        '200':
+          description: new pet record
+          content:
+            application/json:
+              schema:
+                ${'$'}ref: '#/components/schemas/Pet'
+              examples:
+                SUCCESS:
+                  value:
+                    id: 10
+                    name: Archie
+        '400':
+          description: invalid request
+          content:
+            application/json:
+              examples:
+                INVALID:
+                  value:
+                    message: Name must be a strings
+              schema:
+                type: object
+                properties:
+                  message:
+                    type: string
+components:
+  schemas:
+    Pet:
+      type: object
+      required:
+        - id
+        - name
+      properties:
+        name:
+          type: string
+        id:
+          type: integer
+    NewPet:
+      type: object
+      required:
+        - name
+      properties:
+        name:
+          type: string
+""".trimIndent(), "").toFeature()
+
+        var contractInvalidValueReceived = false
+
+        try {
+            contract.executeTests(object : TestExecutor {
+                override fun execute(request: HttpRequest): HttpResponse {
+                    val dataHeaderValue: String? = request.queryParams["data"]
+
+                    if(dataHeaderValue == "hello")
+                        contractInvalidValueReceived = true
+
+                    return HttpResponse(400, body = parsedJSONObject("""{"message": "invalid request"}"""))
                 }
 
                 override fun setServerState(serverState: Map<String, Value>) {