Skip to content

Commit

Permalink
Fixed bug in test generation
Browse files Browse the repository at this point in the history 
The code was completely short-circuiting generated tests in which there
was a breakable cycle. But in order to verify validity of the API
implementation, we need to test at least one level in the cycle before
breaking it.
  • Loading branch information
@joelrosario
joelrosario committed Mar 8, 2023
1 parent 3b84a9e commit 786a4ac
Show file tree
Hide file tree
Showing 2 changed files with 242 additions and 27 deletions.
4 changes: 2 additions & 2 deletions core/src/main/kotlin/in/specmatic/core/Resolver.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,11 +79,11 @@ data class Resolver(
* returnNullOnCycle=true in which case null is returned. Null is never returned if returnNullOnCycle=false.
*/
fun <T> withCyclePrevention(pattern: Pattern, returnNullOnCycle: Boolean = false, toResult: (r: Resolver) -> T) : T? {
val index = cyclePreventionStack.indexOf(pattern)
val count = cyclePreventionStack.filter { it == pattern }.size
val newCyclePreventionStack = cyclePreventionStack.plus(pattern)

try {
if (index >= 0)
if (count > 1)
// Terminate what would otherwise be an infinite cycle.
throw ContractException("Invalid pattern cycle: ${newCyclePreventionStack}", isCycle = true)

Expand Down
265 changes: 240 additions & 25 deletions core/src/test/kotlin/in/specmatic/core/CyclePrevention.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,119 @@ package `in`.specmatic.core

import `in`.specmatic.conversions.OpenApiSpecification
import `in`.specmatic.core.Result.Success
import `in`.specmatic.core.pattern.parsedJSON
import `in`.specmatic.core.value.Value
import `in`.specmatic.stub.HttpStub
import `in`.specmatic.test.TestExecutor
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.RepeatedTest
import org.junit.jupiter.api.Test

class CyclePrevention {
@Test
fun `one level test`() {
val contract = OpenApiSpecification.fromYAML("""
openapi: "3.0.0"
info:
version: 1.0.0
title: Data API
paths:
/data:
post:
description: Get data
requestBody:
content:
application/json:
schema:
${'$'}ref: '#/components/schemas/TopLevel'
responses:
'200':
description: data
components:
schemas:
TopLevel:
nullable: true
type: object
properties:
key:
${'$'}ref: '#/components/schemas/TopLevel'
""".trimIndent(), "").toFeature()

var testCount = 0

contract.executeTests(object: TestExecutor {
override fun execute(request: HttpRequest): HttpResponse {
testCount += 1
println(request.toLogString())
return HttpResponse.OK
}

override fun setServerState(serverState: Map<String, Value>) {
}

})

assertThat(testCount).isEqualTo(4)
}

@Test
fun `two level test`() {
val contract = OpenApiSpecification.fromYAML("""
openapi: "3.0.0"
info:
version: 1.0.0
title: Data API
paths:
/data:
post:
description: Get data
requestBody:
content:
application/json:
schema:
${'$'}ref: '#/components/schemas/TopLevel'
responses:
'200':
description: data
components:
schemas:
TopLevel:
nullable: true
type: object
properties:
key:
${'$'}ref: '#/components/schemas/NextLevel'
NextLevel:
nullable: true
type: object
properties:
subkey:
${'$'}ref: '#/components/schemas/TopLevel'
""".trimIndent(), "").toFeature()

var testCount = 0

contract.executeTests(object: TestExecutor {
override fun execute(request: HttpRequest): HttpResponse {
testCount += 1
println(request.toLogString())
return HttpResponse.OK
}

override fun setServerState(serverState: Map<String, Value>) {
}

})

assertThat(testCount).isEqualTo(8)
}

@Test
@RepeatedTest(5)
fun `test cycle in optional key to circular ref`() {
// key? -> circular-ref-value

val feature = OpenApiSpecification.fromYAML("""
val stubContract = OpenApiSpecification.fromYAML("""
openapi: "3.0.0"
info:
version: 1.0.0
Expand All @@ -38,12 +139,42 @@ class CyclePrevention {
${'$'}ref: '#/components/schemas/TopLevel'
""".trimIndent(), "").toFeature()

val response = HttpStub(feature).use {
it.client.execute(HttpRequest("GET", "/data"))
HttpStub(stubContract).use {
val randomResponse = it.client.execute(HttpRequest("GET", "/data"))
assertThat(stubContract.scenarios.first().let { it.httpResponsePattern.matches(randomResponse, it.resolver) }).isInstanceOf(
Success::class.java)

val rawJSON = """
{
"http-request": {
"method": "GET",
"path": "/data"
},
"http-response": {
"status": 200,
"body": {
"key": {
"key": {
"key": {}
}
}
}
}
}
""".trimIndent()

val expectationSettingResponse = setExpectation(rawJSON, it)

assertThat(expectationSettingResponse.status).isEqualTo(200)
}

assertThat(feature.scenarios.first().let { it.httpResponsePattern.matches(response, it.resolver) }).isInstanceOf(
Success::class.java)
}

private fun setExpectation(rawJSON: String, it: HttpStub): HttpResponse {
val expectation = parsedJSON(rawJSON)
val expectationSettingResponse =
it.client.execute(HttpRequest("POST", "/_specmatic/expectations", body = expectation))
return expectationSettingResponse
}

@Test
Expand Down Expand Up @@ -82,16 +213,38 @@ class CyclePrevention {
- ${'$'}ref: '#/components/schemas/TopLevel'
""".trimIndent(), "").toFeature()

val response = HttpStub(feature).use {
it.client.execute(HttpRequest("GET", "/data"))
HttpStub(feature).use {
val response = it.client.execute(HttpRequest("GET", "/data"))

assertThat(feature.scenarios.first().let { it.httpResponsePattern.matches(response, it.resolver) }).isInstanceOf(
Success::class.java)

assertThat(
setExpectation(
"""
{
"http-request": {
"method": "GET",
"path": "/data"
},
"http-response": {
"status": 200,
"body": {
"key": {
"key": {
"key": null
}
}
}
}
}
""".trimIndent(), it
).status).isEqualTo(200)
}

assertThat(feature.scenarios.first().let { it.httpResponsePattern.matches(response, it.resolver) }).isInstanceOf(
Success::class.java)
}

@Test
// @RepeatedTest(5)
@RepeatedTest(5)
fun `test cycle in optional key to nullable ref`() {
// key? -> circular-ref-value?

Expand All @@ -115,21 +268,62 @@ class CyclePrevention {
schemas:
TopLevel:
type: object
nullable: true
properties:
key:
oneOf:
- type: object
properties: {}
nullable: true
- ${'$'}ref: '#/components/schemas/TopLevel'
${'$'}ref: '#/components/schemas/TopLevel'
""".trimIndent(), "").toFeature()

val response = HttpStub(feature).use {
it.client.execute(HttpRequest("GET", "/data"))
HttpStub(feature).use {
val response = it.client.execute(HttpRequest("GET", "/data"))

assertThat(feature.scenarios.first().let { it.httpResponsePattern.matches(response, it.resolver) }).isInstanceOf(
Success::class.java)

assertThat(
setExpectation(
"""
{
"http-request": {
"method": "GET",
"path": "/data"
},
"http-response": {
"status": 200,
"body": {
"key": {
"key": {
"key": null
}
}
}
}
}
""".trimIndent(), it
).status).isEqualTo(200)

assertThat(
setExpectation(
"""
{
"http-request": {
"method": "GET",
"path": "/data"
},
"http-response": {
"status": 200,
"body": {
"key": {
"key": {
"key": {}
}
}
}
}
}
""".trimIndent(), it
).status).isEqualTo(200)
}

assertThat(feature.scenarios.first().let { it.httpResponsePattern.matches(response, it.resolver) }).isInstanceOf(
Success::class.java)
}

@Test
Expand Down Expand Up @@ -165,11 +359,32 @@ class CyclePrevention {
""".trimIndent(), ""
).toFeature()

val response = HttpStub(feature).use {
it.client.execute(HttpRequest("GET", "/data"))
HttpStub(feature).use {
val response = it.client.execute(HttpRequest("GET", "/data"))
assertThat(response.status).isEqualTo(400)

assertThat(
setExpectation(
"""
{
"http-request": {
"method": "GET",
"path": "/data"
},
"http-response": {
"status": 200,
"body": {
"key": {
"key": {
"key": {}
}
}
}
}
}
""".trimIndent(), it
).status).isEqualTo(400)
}

assertThat(response.status).isEqualTo(400)
}

@Test
Expand Down

0 comments on commit 786a4ac

Please sign in to comment.