From 01f979b2de3983764eb9b13fe0c097f708e5c342 Mon Sep 17 00:00:00 2001 From: Joel Rosario Date: Mon, 6 Mar 2023 21:10:23 +0530 Subject: [PATCH 1/2] Allow invalid request in header for 400 test --- .../in/specmatic/core/HttpRequestPattern.kt | 7 +- .../main/kotlin/in/specmatic/core/Resolver.kt | 32 ++++- .../specmatic/core/pattern/TabularPattern.kt | 18 +-- .../in/specmatic/conversions/OpenApiKtTest.kt | 121 +++++++++++++++++- 4 files changed, 158 insertions(+), 20 deletions(-) diff --git a/core/src/main/kotlin/in/specmatic/core/HttpRequestPattern.kt b/core/src/main/kotlin/in/specmatic/core/HttpRequestPattern.kt index a724ec664..769359b0d 100644 --- a/core/src/main/kotlin/in/specmatic/core/HttpRequestPattern.kt +++ b/core/src/main/kotlin/in/specmatic/core/HttpRequestPattern.kt @@ -366,7 +366,12 @@ data class HttpRequestPattern( } } - fun newBasedOn(row: Row, resolver: Resolver, status: Int = 0): List { + fun newBasedOn(row: Row, initialResolver: Resolver, status: Int = 0): List { + val resolver = if(status in invalidRequestStatuses) + initialResolver.invalidRequestResolver() + else + initialResolver + return attempt(breadCrumb = "REQUEST") { val newURLMatchers = urlMatcher?.newBasedOn(row, resolver) ?: listOf(null) val newBodies: List = attempt(breadCrumb = "BODY") { diff --git a/core/src/main/kotlin/in/specmatic/core/Resolver.kt b/core/src/main/kotlin/in/specmatic/core/Resolver.kt index 8456f2b94..e6a49593a 100644 --- a/core/src/main/kotlin/in/specmatic/core/Resolver.kt +++ b/core/src/main/kotlin/in/specmatic/core/Resolver.kt @@ -5,6 +5,22 @@ import `in`.specmatic.core.value.StringValue import `in`.specmatic.core.value.True import `in`.specmatic.core.value.Value +val actualMatch: (resolver: Resolver, factKey: String?, pattern: Pattern, sampleValue: Value) -> Result = { resolver: Resolver, factKey: String?, pattern: Pattern, sampleValue: Value -> + resolver.actualPatternMatch(factKey, pattern, sampleValue) +} + +val matchAnything: (resolver: Resolver, factKey: String?, pattern: Pattern, sampleValue: Value) -> Result = { resolver: Resolver, factKey: String?, pattern: Pattern, sampleValue: Value -> + Result.Success() +} + +val actualParse: (resolver: Resolver, pattern: Pattern, rowValue: String) -> Value = { resolver: Resolver, pattern: Pattern, rowValue: String -> + resolver.actualParse(pattern, rowValue) +} + +val alwaysReturnStringValue: (resolver: Resolver, pattern: Pattern, rowValue: String) -> Value = { resolver: Resolver, pattern: Pattern, rowValue: String -> + StringValue(rowValue) +} + data class Resolver( val factStore: FactStore = CheckFacts(), val mockMode: Boolean = false, @@ -13,7 +29,9 @@ data class Resolver( val context: Map = emptyMap(), val mismatchMessages: MismatchMessages = DefaultMismatchMessages, val isNegative: Boolean = false, - val generativeTestingEnabled: Boolean = false + val generativeTestingEnabled: Boolean = false, + val patternMatchStrategy: (resolver: Resolver, factKey: String?, pattern: Pattern, sampleValue: Value) -> Result = actualMatch, + val parseStrategy: (resolver: Resolver, pattern: Pattern, rowValue: String) -> Value = actualParse ) { constructor(facts: Map = emptyMap(), mockMode: Boolean = false, newPatterns: Map = emptyMap()) : this(CheckFacts(facts), mockMode, newPatterns) constructor() : this(emptyMap(), false) @@ -37,6 +55,10 @@ data class Resolver( } fun matchesPattern(factKey: String?, pattern: Pattern, sampleValue: Value): Result { + return patternMatchStrategy(this, factKey, pattern, sampleValue) + } + + fun actualPatternMatch(factKey: String?, pattern: Pattern, sampleValue: Value): Result { if (mockMode && sampleValue is StringValue && isPatternToken(sampleValue.string) @@ -95,6 +117,14 @@ data class Resolver( } fun parse(pattern: Pattern, rowValue: String): Value { + return parseStrategy(this, pattern, rowValue) + } + + fun actualParse(pattern: Pattern, rowValue: String): Value { return pattern.parse(rowValue, this) } + + fun invalidRequestResolver(): Resolver { + return this.copy(patternMatchStrategy = matchAnything, parseStrategy = alwaysReturnStringValue) + } } diff --git a/core/src/main/kotlin/in/specmatic/core/pattern/TabularPattern.kt b/core/src/main/kotlin/in/specmatic/core/pattern/TabularPattern.kt index 447a04689..ae93a8c76 100644 --- a/core/src/main/kotlin/in/specmatic/core/pattern/TabularPattern.kt +++ b/core/src/main/kotlin/in/specmatic/core/pattern/TabularPattern.kt @@ -180,7 +180,7 @@ fun newBasedOn(row: Row, key: String, pattern: Pattern, resolver: Resolver): Lis row.containsField(keyWithoutOptionality) -> { val rowValue = row.getField(keyWithoutOptionality) - val fromExamples = if (isPatternToken(rowValue)) { + if (isPatternToken(rowValue)) { val rowPattern = resolver.getPattern(rowValue) attempt(breadCrumb = key) { @@ -194,25 +194,11 @@ fun newBasedOn(row: Row, key: String, pattern: Pattern, resolver: Resolver): Lis resolver.parse(pattern, rowValue) } - when (val matchResult = pattern.matches(parsedRowValue, resolver)) { + when (val matchResult = resolver.matchesPattern(null, pattern, parsedRowValue)) { is Result.Failure -> throw ContractException(matchResult.toFailureReport()) else -> listOf(ExactValuePattern(parsedRowValue)) } } - - fromExamples - -// if(Flags.negativeTestingEnabled()) { -// val vanilla = pattern.newBasedOn(Row(), resolver) -// -// val remainder = vanilla.filterNot { vanillaType -> -// fromExamples.any { item -> vanillaType.encompasses(item, resolver, resolver) is Result.Success } -// } -// -// fromExamples.plus(remainder) -// } else { -// fromExamples -// } } else -> pattern.newBasedOn(row, resolver) } diff --git a/core/src/test/kotlin/in/specmatic/conversions/OpenApiKtTest.kt b/core/src/test/kotlin/in/specmatic/conversions/OpenApiKtTest.kt index fd961d167..096df48a9 100644 --- a/core/src/test/kotlin/in/specmatic/conversions/OpenApiKtTest.kt +++ b/core/src/test/kotlin/in/specmatic/conversions/OpenApiKtTest.kt @@ -1780,7 +1780,7 @@ Feature: Foo API } @Test - fun `contract-invalid test should be allowed for 400 request`() { + fun `contract-invalid test should be allowed for 400 request payload`() { val contract = OpenApiSpecification.fromYAML(""" openapi: "3.0.3" info: @@ -1873,7 +1873,124 @@ components: if(jsonBody.jsonObject["name"] is NumberValue) contractInvalidValueReceived = true - return HttpResponse(422, body = parsedJSONObject("""{"message": "invalid request"}""")) + return HttpResponse(400, body = parsedJSONObject("""{"message": "invalid request"}""")) + } + + override fun setServerState(serverState: Map) { + } + }) + + assertThat(contractInvalidValueReceived).isTrue + } finally { + System.clearProperty(Flags.negativeTestingFlag) + } + } + + @Test + fun `contract-invalid test should be allowed for 400 request header`() { + val contract = OpenApiSpecification.fromYAML(""" +openapi: "3.0.3" +info: + version: 1.0.0 + title: Swagger Petstore + description: A sample API that uses a petstore as an example to demonstrate features in the OpenAPI 3.0 specification + termsOfService: http://swagger.io/terms/ + contact: + name: Swagger API Team + email: apiteam@swagger.io + url: http://swagger.io + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0.html +servers: + - url: http://petstore.swagger.io/api +paths: + /pets: + post: + summary: create a pet + description: Creates a new pet in the store. Duplicates are allowed + operationId: addPet + parameters: + - in: header + name: data + schema: + type: integer + examples: + INVALID: + value: hello + SUCCESS: + value: 10 + requestBody: + description: Pet to add to the store + required: true + content: + application/json: + schema: + ${'$'}ref: '#/components/schemas/NewPet' + examples: + SUCCESS: + value: + name: 'Archie' + INVALID: + value: + name: 10 + responses: + '200': + description: new pet record + content: + application/json: + schema: + ${'$'}ref: '#/components/schemas/Pet' + examples: + SUCCESS: + value: + id: 10 + name: Archie + '400': + description: invalid request + content: + application/json: + examples: + INVALID: + value: + message: Name must be a strings + schema: + type: object + properties: + message: + type: string +components: + schemas: + Pet: + type: object + required: + - id + - name + properties: + name: + type: string + id: + type: integer + NewPet: + type: object + required: + - name + properties: + name: + type: string +""".trimIndent(), "").toFeature() + + var contractInvalidValueReceived = false + + try { + contract.executeTests(object : TestExecutor { + override fun execute(request: HttpRequest): HttpResponse { + val dataHeaderValue: String? = request.headers["data"] + + if(dataHeaderValue == "hello") + contractInvalidValueReceived = true + + return HttpResponse(400, body = parsedJSONObject("""{"message": "invalid request"}""")) } override fun setServerState(serverState: Map) { From c1411ef5d4e84da788b1e0732885b7b3c8850876 Mon Sep 17 00:00:00 2001 From: Joel Rosario Date: Mon, 6 Mar 2023 21:13:08 +0530 Subject: [PATCH 2/2] Added test for invalid query param in 400 test --- .../in/specmatic/conversions/OpenApiKtTest.kt | 104 +++++++++++++++++- 1 file changed, 103 insertions(+), 1 deletion(-) diff --git a/core/src/test/kotlin/in/specmatic/conversions/OpenApiKtTest.kt b/core/src/test/kotlin/in/specmatic/conversions/OpenApiKtTest.kt index 096df48a9..1c2a3a129 100644 --- a/core/src/test/kotlin/in/specmatic/conversions/OpenApiKtTest.kt +++ b/core/src/test/kotlin/in/specmatic/conversions/OpenApiKtTest.kt @@ -1887,7 +1887,7 @@ components: } @Test - fun `contract-invalid test should be allowed for 400 request header`() { + fun `contract-invalid test should be allowed for 400 query parameter`() { val contract = OpenApiSpecification.fromYAML(""" openapi: "3.0.3" info: @@ -2003,6 +2003,108 @@ components: } } + @Test + fun `contract-invalid test should be allowed for 400 request header`() { + val contract = OpenApiSpecification.fromYAML(""" +openapi: "3.0.3" +info: + version: 1.0.0 + title: Swagger Petstore + description: A sample API that uses a petstore as an example to demonstrate features in the OpenAPI 3.0 specification + termsOfService: http://swagger.io/terms/ + contact: + name: Swagger API Team + email: apiteam@swagger.io + url: http://swagger.io + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0.html +servers: + - url: http://petstore.swagger.io/api +paths: + /pets: + get: + summary: query for a pet + description: Queries info on a pet + parameters: + - in: query + name: data + schema: + type: integer + examples: + INVALID: + value: hello + SUCCESS: + value: 10 + responses: + '200': + description: new pet record + content: + application/json: + schema: + ${'$'}ref: '#/components/schemas/Pet' + examples: + SUCCESS: + value: + id: 10 + name: Archie + '400': + description: invalid request + content: + application/json: + examples: + INVALID: + value: + message: Name must be a strings + schema: + type: object + properties: + message: + type: string +components: + schemas: + Pet: + type: object + required: + - id + - name + properties: + name: + type: string + id: + type: integer + NewPet: + type: object + required: + - name + properties: + name: + type: string +""".trimIndent(), "").toFeature() + + var contractInvalidValueReceived = false + + try { + contract.executeTests(object : TestExecutor { + override fun execute(request: HttpRequest): HttpResponse { + val dataHeaderValue: String? = request.queryParams["data"] + + if(dataHeaderValue == "hello") + contractInvalidValueReceived = true + + return HttpResponse(400, body = parsedJSONObject("""{"message": "invalid request"}""")) + } + + override fun setServerState(serverState: Map) { + } + }) + + assertThat(contractInvalidValueReceived).isTrue + } finally { + System.clearProperty(Flags.negativeTestingFlag) + } + } + @Test fun `a test marked WIP should setup the scenario to ignore failure`() { val contract = OpenApiSpecification.fromYAML("""