Evaluate payload body in prologues and epilogues.

CHANGELOG.md

@@ -46,6 +46,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added the ability to skip an individual chapter test ([#765](https://github.com/opensearch-project/opensearch-api-specification/pull/765))
 - Added uploading of test spec logs ([#767](https://github.com/opensearch-project/opensearch-api-specification/pull/767))
 - Added `POST /_plugins/_ml/memory`, `POST /_plugins/_ml/memory/_search`, `{memory_id}/_search`, `{memory_id}/messages`, `PUT /_plugins/_ml/memory/{memory_id}`, `message/{message_id}`,  `GET /_plugins/_ml/memory`, `GET /_plugins/_ml/memory/{memory_id}`, `_search`, `message/{message_id}`, `{memory_id}/messages`, `{memory_id}/_search`, `message/{message_id}/traces`,  and `DELETE /_plugins/_ml/memory/{memory_id}` ([#771](https://github.com/opensearch-project/opensearch-api-specification/pull/771))
+- Added support for evaluating response payloads in prologues and epilogues ([#772](https://github.com/opensearch-project/opensearch-api-specification/pull/772))
 
 ### Removed
 - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652))

tools/src/tester/ChapterEvaluator.ts

@@ -16,12 +16,12 @@ import type OperationLocator from './OperationLocator'
 import type SchemaValidator from './SchemaValidator'
 import { type StoryOutputs } from './StoryOutputs'
 import { ChapterOutput } from './ChapterOutput'
-import { Operation, atomizeChangeset, diff } from 'json-diff-ts'
 import _ from 'lodash'
 import { Logger } from 'Logger'
 import { sleep, to_json } from '../helpers'
-import { APPLICATION_JSON } from "./MimeTypes";
+import { APPLICATION_JSON } from "./MimeTypes"
 import { ParsedChapter } from './types/parsed_story.types'
+import ResponsePayloadEvaluator from './ResponsePayloadEvaluator'
 
 export default class ChapterEvaluator {
   private readonly logger: Logger
@@ -70,7 +70,7 @@ export default class ChapterEvaluator {
     const payload_schema_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_schema(chapter, response, operation) : { result: Result.SKIPPED }
     const output_values_evaluation: EvaluationWithOutput = status.result === Result.PASSED ? ChapterOutput.extract_output_values(response, chapter.output) : { evaluation: { result: Result.SKIPPED } }
     const response_payload: Payload | undefined = status.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.payload) : chapter.response?.payload
-    const payload_body_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_body(response, response_payload) : { result: Result.SKIPPED }
+    const payload_body_evaluation = status.result === Result.PASSED ? new ResponsePayloadEvaluator(this.logger).evaluate(response, response_payload) : { result: Result.SKIPPED }
 
     if (output_values_evaluation.output) this.logger.info(`$ ${to_json(output_values_evaluation.output)}`)
 
@@ -151,22 +151,6 @@ export default class ChapterEvaluator {
     return result
   }
 
-  #evaluate_payload_body(response: ActualResponse, expected_payload?: Payload): Evaluation {
-    if (expected_payload == null) return { result: Result.PASSED }
-    const payload = response.payload
-    this.logger.info(`${to_json(payload)}`)
-    const delta = atomizeChangeset(diff(expected_payload, payload))
-    const messages: string[] = _.compact(delta.map((value, _index, _array) => {
-      switch (value.type) {
-        case Operation.UPDATE:
-          return `expected ${value.path.replace('$.', '')}='${value.oldValue}', got '${value.value}'`
-        case Operation.REMOVE:
-          return `missing ${value.path.replace('$.', '')}='${value.value}'`
-      }
-    }))
-    return messages.length > 0 ? { result: Result.FAILED, message: _.join(messages, ', ') } : { result: Result.PASSED }
-  }
-
   #evaluate_payload_schema(chapter: ParsedChapter, response: ActualResponse, operation: ParsedOperation): Evaluation {
     const content_type = chapter.response?.content_type ?? APPLICATION_JSON
 

tools/src/tester/ResponsePayloadEvaluator.ts

@@ -0,0 +1,39 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+
+import _ from "lodash"
+import { Evaluation, Result } from './types/eval.types'
+import { Logger } from "../Logger"
+import { to_json } from "../helpers"
+import { ActualResponse, Payload } from "./types/story.types"
+import { atomizeChangeset, diff, Operation } from "json-diff-ts"
+
+export default class ResponsePayloadEvaluator {
+  private readonly logger: Logger
+
+  constructor(logger: Logger) {
+    this.logger = logger
+  }
+
+  evaluate(response: ActualResponse, expected_payload?: Payload): Evaluation {
+    if (expected_payload == null) return { result: Result.PASSED }
+    const payload = response.payload
+    this.logger.info(`${to_json(payload)}`)
+    const delta = atomizeChangeset(diff(expected_payload, payload))
+    const messages: string[] = _.compact(delta.map((value, _index, _array) => {
+      switch (value.type) {
+        case Operation.UPDATE:
+          return `expected ${value.path.replace('$.', '')}='${value.oldValue}', got '${value.value}'`
+        case Operation.REMOVE:
+          return `missing ${value.path.replace('$.', '')}='${value.value}'`
+      }
+    }))
+    return messages.length > 0 ? { result: Result.FAILED, message: _.join(messages, ', ') } : { result: Result.PASSED }
+  }
+}

tools/src/tester/SupplementalChapterEvaluator.ts

@@ -7,22 +7,23 @@
 * compatible open source license.
 */
 
-import _ from "lodash";
-import { ChapterOutput } from "./ChapterOutput";
-import ChapterReader from "./ChapterReader";
-import { StoryOutputs } from "./StoryOutputs";
-import { overall_result } from "./helpers";
-import { ChapterEvaluation, EvaluationWithOutput, Result } from './types/eval.types';
-import { Logger } from "../Logger";
-import { sleep, to_json } from "../helpers";
-import { SupplementalChapter } from "./types/story.types";
+import _ from "lodash"
+import { ChapterOutput } from "./ChapterOutput"
+import ChapterReader from "./ChapterReader"
+import { StoryOutputs } from "./StoryOutputs"
+import { overall_result } from "./helpers"
+import { ChapterEvaluation, EvaluationWithOutput, Result } from './types/eval.types'
+import { Logger } from "../Logger"
+import { sleep, to_json } from "../helpers"
+import { Payload, SupplementalChapter } from "./types/story.types"
+import ResponsePayloadEvaluator from './ResponsePayloadEvaluator'
 
 export default class SupplementalChapterEvaluator {
-  private readonly _chapter_reader: ChapterReader;
-  private readonly logger: Logger;
+  private readonly _chapter_reader: ChapterReader
+  private readonly logger: Logger
 
   constructor(chapter_reader: ChapterReader, logger: Logger) {
-    this._chapter_reader = chapter_reader;
+    this._chapter_reader = chapter_reader
     this.logger = logger
   }
 
@@ -49,10 +50,11 @@ export default class SupplementalChapterEvaluator {
     const response = await this._chapter_reader.read(chapter, story_outputs)
     const output_values_evaluation = ChapterOutput.extract_output_values(response, chapter.output)
     if (output_values_evaluation.output) this.logger.info(`$ ${to_json(output_values_evaluation.output)}`)
-
     const status = chapter.status ?? [200, 201]
     const overall = status.includes(response.status) ? { result: Result.PASSED } : { result: Result.ERROR, message: response.message, error: response.error as Error }
-    const result: Result = overall_result(_.compact([overall, output_values_evaluation.evaluation]))
+    const response_payload: Payload | undefined = overall.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.payload) : chapter.response?.payload
+    const payload_body_evaluation = overall.result === Result.PASSED ? new ResponsePayloadEvaluator(this.logger).evaluate(response, response_payload) : { result: Result.SKIPPED }
+    const result: Result = overall_result(_.compact([overall, payload_body_evaluation, output_values_evaluation.evaluation]))
 
     var evaluation_result: EvaluationWithOutput = { evaluation: { result } }
     if (output_values_evaluation.output) { evaluation_result.output = output_values_evaluation.output }

tools/src/tester/types/story.types.ts

@@ -63,23 +63,6 @@ export type Version = string;
  * via the `definition` "DistributionsList".
  */
 export type DistributionsList = string[];
-/**
- * Number of times to retry on error.
- *
- *
- * This interface was referenced by `Story`'s JSON-Schema
- * via the `definition` "Retry".
- */
-export type Retry = {
-  /**
-   * Number of retries.
-   */
-  count: number;
-  /**
-   * Number of milliseconds to wait before retrying.
-   */
-  wait?: number;
-};
 /**
  * This interface was referenced by `Story`'s JSON-Schema
  * via the `definition` "Chapter".
@@ -90,6 +73,9 @@ export type Chapter = ChapterRequest & {
    * A brief description of the chapter.
    */
   synopsis: string;
+  /**
+   * An explanation is provided to clarify why it has been skipped.
+   */
   pending?: string;
   response?: ExpectedResponse;
   warnings?: Warnings;
@@ -131,6 +117,7 @@ export interface ChapterRequest {
   version?: Version;
   distributions?: Distributions;
   retry?: Retry;
+  response?: ExpectedResponse;
 }
 /**
  * This interface was referenced by `Story`'s JSON-Schema
@@ -177,6 +164,23 @@ export interface Distributions {
   included?: DistributionsList;
   excluded?: DistributionsList;
 }
+/**
+ * Number of times to retry on error.
+ *
+ *
+ * This interface was referenced by `Story`'s JSON-Schema
+ * via the `definition` "Retry".
+ */
+export interface Retry {
+  /**
+   * Number of retries.
+   */
+  count: number;
+  /**
+   * Number of milliseconds to wait before retrying.
+   */
+  wait?: number;
+}
 /**
  * This interface was referenced by `Story`'s JSON-Schema
  * via the `definition` "ExpectedResponse".

tools/tests/tester/ResponsePayloadEvaluator.test.ts

@@ -0,0 +1,39 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+
+import { Result } from "tester/types/eval.types";
+import ResponsePayloadEvaluator from "tester/ResponsePayloadEvaluator";
+import { Logger } from "Logger";
+import { ActualResponse } from "tester/types/story.types";
+
+function create_response(payload: any): ActualResponse {
+  return {
+    status: 200,
+    content_type: 'application/json',
+    payload
+  }
+}
+
+describe('ResponsePayloadEvaluator', () => {
+  const evaluator = new ResponsePayloadEvaluator(new Logger())
+
+  describe('evaluate', () => {
+    test('succeeds without an expected payload', () => {
+      expect(evaluator.evaluate(create_response({}), undefined)).toEqual({ result: Result.PASSED })
+    })
+
+    test('fails with a non-matching payload', () => {
+      expect(evaluator.evaluate(create_response({}), { x: 1 })).toEqual({ result: Result.FAILED, message: "missing x='1'" })
+    })
+
+    test('succeeds with a matching payload', () => {
+      expect(evaluator.evaluate(create_response({ x: 1 }), { x: 1 })).toEqual({ result: Result.PASSED })
+    })
+  })
+})

tools/tests/tester/SupplementalChapterEvaluator.test.ts

@@ -86,5 +86,53 @@ describe('SupplementalChapterEvaluator', () => {
       expect(result.overall.result).toEqual(Result.ERROR)
       expect(count).toEqual(5)
     })
+
+    test('a valid response payload', async () => {
+      mock.onAny().reply(200, '{"acknowledged":true}', { "content-type": "application/json" })
+
+      expect(
+        await supplemental_chapter_evaluator.evaluate({
+          path: '/test',
+          method: 'PUT',
+          request: {
+            payload: {}
+          },
+          response: {
+            status: 200,
+            payload: {
+              acknowledged: true
+            }
+          }
+        }, story_outputs)).toEqual({
+        title: 'PUT /test',
+        overall: {
+          result: Result.PASSED
+        }
+      })
+    })
+
+    test('an invalid response payload', async () => {
+      mock.onAny().reply(200, '{"acknowledged":false}', { "content-type": "application/json" })
+
+      expect(
+        await supplemental_chapter_evaluator.evaluate({
+          path: '/test',
+          method: 'PUT',
+          request: {
+            payload: {}
+          },
+          response: {
+            status: 200,
+            payload: {
+              acknowledged: true
+            }
+          }
+        }, story_outputs)).toEqual({
+        title: 'PUT /test',
+        overall: {
+          result: Result.FAILED
+        }
+      })
+    })
   })
 })