diff --git a/TrafficCapture/README.md b/TrafficCapture/README.md index 93f4e8e89..1b424572e 100644 --- a/TrafficCapture/README.md +++ b/TrafficCapture/README.md @@ -110,27 +110,46 @@ The body of the messages is sometimes gzipped which makes it difficult to repres and responses is base64 encoded before it is logged. This makes the files stable, but not human-readable. We have provided a utility script that can parse these files and output them to a human-readable format: the bodies are -base64 decoded, un-gzipped if applicable, and parsed as JSON if applicable. They're then saved back to JSON format on disk. +base64 decoded and parsed as JSON if applicable. They're then saved back to JSON format to stdout or file. To use this utility from the Migration Console, ```sh -$ ./humanReadableLogs.py --help -usage: humanReadableLogs.py [-h] [--outfile OUTFILE] infile - -positional arguments: - infile Path to input logged tuple file. - -options: - -h, --help show this help message and exit - --outfile OUTFILE Path for output human readable tuple file. - -# By default, the output file is the same path as the input file, but the file name is prefixed with `readable-`. -$ ./humanReadableLogs.py /shared_replayer_output/tuples.log -Input file: /shared_replayer_output/tuples.log; Output file: /shared_replayer_output/readable-tuples.log - +$ console tuples show --help +Usage: console tuples convert [OPTIONS] + +Options: + --in FILENAME + --out FILENAME + --help Show this message and exit. + +# By default, the input and output files are `stdin` and `stdout` respectively, so they can be piped together with other tools. +$ console tuples show --in /shared-logs-output/traffic-replayer-default/86ca83e66197/tuples/mini_tuples.log | jq +{ + "sourceRequest": { + "Request-URI": "/", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "User-Agent": "python-requests/2.32.3", + "Accept-Encoding": "gzip, deflate, zstd", + "Accept": "*/*", + "Connection": "keep-alive", + "Authorization": "Basic YWRtaW46YWRtaW4=", + "body": "" + }, + "sourceResponse": { + "HTTP-Version": { + "keepAliveDefault": true + }, + "Status-Code": 200, + "Reason-Phrase": "OK", + ... + }, + ... +} # A specific output file can also be specified. -$ ./humanReadableLogs.py /shared_replayer_output/tuples.log --outfile local-tuples.log -Input file: /shared_replayer_output/tuples.log; Output file: local-tuples.log +$ console tuples show --in /shared_replayer_output/tuples.log --out local-tuples.log +Converted tuples output to local-tuples.log ``` ### Capture Kafka Offloader diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile index 15f1e0ee2..ab02505f2 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile @@ -27,9 +27,6 @@ COPY osiPipelineTemplate.yaml /root/ COPY msk-iam-auth.properties /root/kafka-tools/aws COPY kafkaCmdRef.md /root/kafka-tools -COPY humanReadableLogs.py /root/ -RUN chmod ug+x /root/humanReadableLogs.py - COPY showFetchMigrationCommand.sh /root/ RUN chmod ug+x /root/showFetchMigrationCommand.sh diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/humanReadableLogs.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/humanReadableLogs.py deleted file mode 100755 index e6c1a3130..000000000 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/humanReadableLogs.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import base64 -import gzip -import json -import pathlib -from typing import Optional -import logging - -from tqdm import tqdm -from tqdm.contrib.logging import logging_redirect_tqdm - -logger = logging.getLogger(__name__) - -BASE64_ENCODED_TUPLE_PATHS = ["sourceRequest.body", "targetRequest.body", "sourceResponse.body"] -# TODO: I'm not positive about the capitalization of the Content-Encoding and Content-Type headers. -# This version worked on my test cases, but not guaranteed to work in all cases. -CONTENT_ENCODING_PATH = { - BASE64_ENCODED_TUPLE_PATHS[0]: "sourceRequest.Content-Encoding", - BASE64_ENCODED_TUPLE_PATHS[1]: "targetRequest.Content-Encoding", - BASE64_ENCODED_TUPLE_PATHS[2]: "sourceResponse.Content-Encoding" -} -CONTENT_TYPE_PATH = { - BASE64_ENCODED_TUPLE_PATHS[0]: "sourceRequest.Content-Type", - BASE64_ENCODED_TUPLE_PATHS[1]: "targetRequest.Content-Type", - BASE64_ENCODED_TUPLE_PATHS[2]: "sourceResponse.Content-Type" -} -TRANSFER_ENCODING_PATH = { - BASE64_ENCODED_TUPLE_PATHS[0]: "sourceRequest.Transfer-Encoding", - BASE64_ENCODED_TUPLE_PATHS[1]: "targetRequest.Transfer-Encoding", - BASE64_ENCODED_TUPLE_PATHS[2]: "sourceResponse.Transfer-Encoding" -} - -CONTENT_TYPE_JSON = "application/json" -CONTENT_ENCODING_GZIP = "gzip" -TRANSFER_ENCODING_CHUNKED = "chunked" -URI_PATH = "sourceRequest.Request-URI" -BULK_URI_PATH = "_bulk" - - -class DictionaryPathException(Exception): - pass - - -def get_element(element: str, dict_: dict, raise_on_error=False, try_lowercase_keys=False) -> Optional[any]: - """This has a limited version of case-insensitivity. It specifically only checks the provided key - and an all lower-case version of the key (if `try_lowercase_keys` is True).""" - keys = element.split('.') - rv = dict_ - for key in keys: - try: - if key in rv: - rv = rv[key] - continue - if try_lowercase_keys and key.lower() in rv: - rv = rv[key.lower()] - except KeyError: - if raise_on_error: - raise DictionaryPathException(f"Key {key} was not present.") - else: - return None - return rv - - -def set_element(element: str, dict_: dict, value: any) -> None: - keys = element.split('.') - rv = dict_ - for key in keys[:-1]: - rv = rv[key] - rv[keys[-1]] = value - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument("infile", type=pathlib.Path, help="Path to input logged tuple file.") - parser.add_argument("--outfile", type=pathlib.Path, help="Path for output human readable tuple file.") - return parser.parse_args() - - -def decode_chunked(data: bytes) -> bytes: - newdata = [] - next_newline = data.index(b'\r\n') - chunk = data[next_newline + 2:] - while len(chunk) > 7: # the final EOM chunk is 7 bytes - next_newline = chunk.index(b'\r\n') - newdata.append(chunk[:next_newline]) - chunk = chunk[next_newline + 2:] - return b''.join(newdata) - - -def parse_body_value(raw_value: str, content_encoding: Optional[str], - content_type: Optional[str], is_bulk: bool, is_chunked_transfer: bool, line_no: int): - # Body is base64 decoded - try: - b64decoded = base64.b64decode(raw_value) - except Exception as e: - logger.error(f"Body value on line {line_no} could not be decoded: {e}. Skipping parsing body value.") - return None - - # Decoded data is un-chunked, if applicable - if is_chunked_transfer: - contiguous_data = decode_chunked(b64decoded) - else: - contiguous_data = b64decoded - - # Data is un-gzipped, if applicable - is_gzipped = content_encoding is not None and content_encoding == CONTENT_ENCODING_GZIP - if is_gzipped: - try: - unzipped = gzip.decompress(contiguous_data) - except Exception as e: - logger.error(f"Body value on line {line_no} should be gzipped but could not be unzipped: {e}. " - "Skipping parsing body value.") - return contiguous_data - else: - unzipped = contiguous_data - - # Data is decoded to utf-8 string - try: - decoded = unzipped.decode("utf-8") - except Exception as e: - logger.error(f"Body value on line {line_no} could not be decoded to utf-8: {e}. " - "Skipping parsing body value.") - return unzipped - - # Data is parsed as json, if applicable - is_json = content_type is not None and CONTENT_TYPE_JSON in content_type - if is_json and len(decoded) > 0: - # Data is parsed as a bulk json, if applicable - if is_bulk: - try: - return [json.loads(line) for line in decoded.splitlines()] - except Exception as e: - logger.error("Body value on line {line_no} should be a bulk json (list of json lines) but " - f"could not be parsed: {e}. Skipping parsing body value.") - return decoded - try: - return json.loads(decoded) - except Exception as e: - logger.error(f"Body value on line {line_no} should be a json but could not be parsed: {e}. " - "Skipping parsing body value.") - return decoded - return decoded - - -def parse_tuple(line: str, line_no: int) -> dict: - tuple = json.loads(line) - try: - is_bulk_path = BULK_URI_PATH in get_element(URI_PATH, tuple, raise_on_error=True) - except DictionaryPathException as e: - logger.error(f"`{URI_PATH}` on line {line_no} could not be loaded: {e} " - f"Skipping parsing tuple.") - return tuple - for body_path in BASE64_ENCODED_TUPLE_PATHS: - base64value = get_element(body_path, tuple) - if base64value is None: - # This component has no body element, which is potentially valid. - continue - value = decode_base64_http_message(base64value, CONTENT_ENCODING_PATH[body_path], CONTENT_TYPE_PATH[body_path], - TRANSFER_ENCODING_PATH[body_path], is_bulk_path, line_no, tuple) - if value and type(value) is not bytes: - set_element(body_path, tuple, value) - for target_response in get_element("targetResponses", tuple): - value = decode_base64_http_message(base64value, "Content-Encoding", "Content-Type", - "Transfer-Encoding", is_bulk_path, line_no, target_response) - if value and type(value) is not bytes: - set_element("body", target_response, value) - return tuple - - -def decode_base64_http_message(base64value, content_encoding, content_type, transfer_encoding, - is_bulk_path, line_no, tuple): - content_encoding = get_element(content_encoding, tuple, try_lowercase_keys=True) - content_type = get_element(content_type, tuple, try_lowercase_keys=True) - is_chunked_transfer = get_element(transfer_encoding, - tuple, try_lowercase_keys=True) == TRANSFER_ENCODING_CHUNKED - return parse_body_value(base64value, content_encoding, content_type, is_bulk_path, - is_chunked_transfer, line_no) - - -if __name__ == "__main__": - args = parse_args() - if args.outfile: - outfile = args.outfile - else: - outfile = args.infile.parent / f"readable-{args.infile.name}" - print(f"Input file: {args.infile}; Output file: {outfile}") - - logging.basicConfig(level=logging.INFO) - with logging_redirect_tqdm(): - with open(args.infile, 'r') as in_f: - with open(outfile, 'w') as out_f: - for i, line in tqdm(enumerate(in_f)): - print(json.dumps(parse_tuple(line, i + 1)), file=out_f) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/cli.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/cli.py index c7f5b0d01..aa9a9544a 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/cli.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/cli.py @@ -1,5 +1,6 @@ import json from pprint import pprint +import sys import click import console_link.middleware.clusters as clusters_ import console_link.middleware.metrics as metrics_ @@ -8,6 +9,7 @@ import console_link.middleware.metadata as metadata_ import console_link.middleware.replay as replay_ import console_link.middleware.kafka as kafka_ +import console_link.middleware.tuples as tuples_ from console_link.models.utils import ExitCode from console_link.environment import Environment @@ -485,6 +487,26 @@ def completion(ctx, config_file, json, shell): ctx.exit(1) +@cli.group(name="tuples") +@click.pass_obj +def tuples_group(ctx): + """ All commands related to tuples. """ + pass + + +@tuples_group.command() +@click.option('--in', 'inputfile', + type=click.File('r'), + default=sys.stdin) +@click.option('--out', 'outputfile', + type=click.File('a'), + default=sys.stdout) +def show(inputfile, outputfile): + tuples_.convert(inputfile, outputfile) + if outputfile != sys.stdout: + click.echo(f"Converted tuples output to {outputfile.name}") + + ################################################# if __name__ == "__main__": diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/middleware/tuples.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/middleware/tuples.py new file mode 100644 index 000000000..ba8363c34 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/middleware/tuples.py @@ -0,0 +1,10 @@ +import logging +import typing + +from console_link.models.tuple_reader import TupleReader +logger = logging.getLogger(__name__) + + +def convert(inputfile: typing.TextIO, ouptutfile: typing.TextIO): + tuple_reader = TupleReader() + tuple_reader.transform_stream(inputfile, ouptutfile) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/tuple_reader.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/tuple_reader.py new file mode 100644 index 000000000..f055c5c6b --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/tuple_reader.py @@ -0,0 +1,214 @@ +from enum import Enum +import re + +import json +from typing import Any, Dict, Generator, List, Self, Set, TextIO, Union +import base64 +from typing import Optional + +import logging + +logger = logging.getLogger(__name__) + + +class TupleReader: + """ This class is fairly minimal for now. There is likely a future in which multiple + tuple storage locations/types are supported, but we are not there yet and don't have + a clear enough vision for it to make sense to frame it out now.""" + def __init__(self) -> None: + # Initialize a TupleReader object. + pass + + def transform_stream(self, inputfile: TextIO, outputfile: TextIO): + transformer = self._transform_lines(inputfile.readlines()) + while True: + try: + json.dump(next(transformer), outputfile) + outputfile.write('\n') + except StopIteration: + logger.info("Reached the end of the input object") + return + + def _transform_lines(self, lines: List[str]) -> Generator[Dict, None, None]: + for i, line in enumerate(lines): + yield parse_tuple(line, i + 1) + + +CONTENT_TYPE_JSON = "application/json" +BULK_URI_PATH = "_bulk" + +SOURCE_REQUEST = "sourceRequest" +TARGET_REQUEST = "targetRequest" +SOURCE_RESPONSE = "sourceResponse" +TARGET_RESPONSE = "targetResponses" + +SINGLE_COMPONENTS = [SOURCE_REQUEST, SOURCE_RESPONSE, TARGET_REQUEST] +LIST_COMPONENTS = [TARGET_RESPONSE] + +URI_PATH = SOURCE_REQUEST + ".Request-URI" + +CONTENT_TYPE_REGEX = re.compile('Content-Type', flags=re.IGNORECASE) + + +class DictionaryPathException(Exception): + pass + + +def get_element_with_regex(regex: re.Pattern, dict_: Dict, raise_on_error=False): + keys = dict_.keys() + try: + match = next(filter(regex.match, keys)) + except StopIteration: + if raise_on_error: + raise DictionaryPathException(f"An element matching the regex ({regex}) was not found.") + return None + + return dict_[match] + + +def get_element(element: str, dict_: dict, raise_on_error=False, try_lowercase_keys=False) -> Optional[any]: + """This has a limited version of case-insensitivity. It specifically only checks the provided key + and an all lower-case version of the key (if `try_lowercase_keys` is True).""" + keys = element.split('.') + rv = dict_ + for key in keys: + try: + if key in rv: + rv = rv[key] + continue + elif try_lowercase_keys and key.lower() in rv: + rv = rv[key.lower()] + else: + raise KeyError + except KeyError: + if raise_on_error: + raise DictionaryPathException(f"Key {key} was not present.") + else: + return None + return rv + + +def set_element(element: str, dict_: dict, value: any) -> None: + keys = element.split('.') + rv = dict_ + for key in keys[:-1]: + try: + rv = rv[key] + except KeyError: + raise DictionaryPathException(f"Key {key} was not present.") + try: + rv[keys[-1]] = value + except TypeError: + raise DictionaryPathException(f"Path {element} did not reach an assignable object.") + + +Flag = Enum('Flag', ['Bulk_Request', 'Json']) + + +class TupleComponent: + def __init__(self, component_name: str, component: Dict, line_no: int, is_bulk_path: bool): + body = get_element("body", component) + self.value: Union[bytes, str] = body + + self.flags = get_flags_for_component(component, is_bulk_path) + + self.line_no = line_no + self.component_name = component_name + + self.final_value: Union[Dict, List, str, bytes] = {} + self.error = False + + def b64decode(self) -> Self: + if self.error or self.value is None: + return self + try: + self.value = base64.b64decode(self.value) + except Exception as e: + self.error = (f"Body value of {self.component_name} on line {self.line_no} could not be decoded: {e}." + "Skipping parsing body value.") + logger.debug(self.error) + logger.debug(self.value) + return self + + def decode_utf8(self) -> Self: + if self.error or self.value is None: + return self + try: + self.value = self.value.decode("utf-8") + except Exception as e: + self.error = (f"Body value of {self.component_name} on line {self.line_no} could not be decoded to utf-8: " + f"{e}. Skipping parsing body value.") + logger.debug(self.error) + logger.debug(self.value) + return self + + def parse_as_json(self) -> Self: + if self.error or self.value is None: + return self + if Flag.Json not in self.flags: + self.final_value = self.value + return self + + if self.value.strip() == "": + self.final_value = self.value + return self + + if Flag.Bulk_Request in self.flags: + try: + self.final_value = [json.loads(line) for line in self.value.splitlines()] + except Exception as e: + self.error = (f"Body value of {self.component_name} on line {self.line_no} should be a bulk json, but " + f"could not be parsed: {e}. Skipping parsing body value.") + logger.debug(self.error) + logger.debug(self.value) + self.final_value = self.value + else: + try: + self.final_value = json.loads(self.value) + except Exception as e: + self.error = (f"Body value of {self.component_name} on line {self.line_no} should be a json, but " + f"could not be parsed: {e}. Skipping parsing body value.") + logger.debug(self.error) + logger.debug(self.value) + self.final_value = self.value + return self + + +def get_flags_for_component(component: Dict[str, Any], is_bulk_path: bool) -> Set[Flag]: + content_type = get_element_with_regex(CONTENT_TYPE_REGEX, component) + is_json = content_type is not None and CONTENT_TYPE_JSON in content_type + return {Flag.Json if is_json else None, + Flag.Bulk_Request if is_bulk_path else None} - {None} + + +def parse_tuple(line: str, line_no: int) -> dict: + initial_tuple = json.loads(line) + try: + is_bulk_path = BULK_URI_PATH in get_element(URI_PATH, initial_tuple, raise_on_error=True) + except DictionaryPathException as e: + logger.error(f"`{URI_PATH}` on line {line_no} could not be loaded: {e} " + f"Skipping parsing tuple.") + return initial_tuple + + for component in SINGLE_COMPONENTS: + tuple_component = TupleComponent(component, initial_tuple[component], line_no, is_bulk_path) + + processed_tuple = tuple_component.b64decode().decode_utf8().parse_as_json() + final_value = processed_tuple.final_value + if not processed_tuple.error: + set_element(component + ".body", initial_tuple, final_value) + else: + logger.error(processed_tuple.error) + + for component in LIST_COMPONENTS: + for i, item in enumerate(initial_tuple[component]): + tuple_component = TupleComponent(f"{component} item {i}", item, line_no, is_bulk_path) + + processed_tuple = tuple_component.b64decode().decode_utf8().parse_as_json() + final_value = processed_tuple.final_value + if not processed_tuple.error: + set_element("body", item, final_value) + else: + logger.error(processed_tuple.error) + + return initial_tuple diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/invalid_tuple.json b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/invalid_tuple.json new file mode 100644 index 000000000..7960fdfda --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/invalid_tuple.json @@ -0,0 +1,51 @@ +{ + "sourceRequest": { + "Request-URI": "/_cat/indices?v", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "Authorization": "Basic YWRtaW46YWRtaW4=", + "User-Agent": "curl/8.5.0", + "Accept": "*/*", + "body": "" + }, + "sourceResponse": { + "HTTP-Version": { + "keepAliveDefault": true + }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 59, + "content-type": "text/plain; charset=UTF-8", + "content-length": "214", + "comment": "This body has a few characters modified to make it un-decodeable as utf-8", + "body": "aGVhbHRoIHN0YXR1cyBpbmRleCAgICACB1dWlkICAgICAgICAgICAgICAgICAgIHByaSByZXAgZG9jcy5jb3VudCBkb2NzLmRlbGV0ZWQgc3RvcmUuc2l6ZSBwcmkuc3RvcmUuc2l6ZQpncmVlbiAgb3BlbiAgIHNlYXJjaGd1YXJkIHlKS1hQTUh0VFJPTklYU1pYQ193bVEgICAxICAgMCAgICAgICAgICA4ICAgICAgICAgICAgMCAgICAgNDQuN2tiICAgICAgICAgNDQuN2tiCg==" + }, + "targetRequest": { + "Request-URI": "/_cat/indices?v", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "Authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "User-Agent": "curl/8.5.0", + "Accept": "*/*", + "body": "" + }, + "targetResponses": [ + { + "HTTP-Version": { + "keepAliveDefault": true + }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 721, + "content-type": "application/json; charset=UTF-8", + "content-length": "484", + "comment": "This response is valid text, but is not valid as json, despite the `content-type` header", + "body": "aGVhbHRoIHN0YXR1cyBpbmRleCAgICAgICAgICAgICAgICAgICAgIHV1aWQgICAgICAgICAgICAgICAgICAgcHJpIHJlcCBkb2NzLmNvdW50IGRvY3MuZGVsZXRlZCBzdG9yZS5zaXplIHByaS5zdG9yZS5zaXplCmdyZWVuICBvcGVuICAgLm9wZW5zZWFyY2gtb2JzZXJ2YWJpbGl0eSA4Vy1vWUhmYlN5U3JkeFFFX3NPbnpnICAgMSAgIDAgICAgICAgICAgMCAgICAgICAgICAgIDAgICAgICAgMjA4YiAgICAgICAgICAgMjA4YgpncmVlbiAgb3BlbiAgIC5wbHVnaW5zLW1sLWNvbmZpZyAgICAgICAgRjludnh2c2dSelNibG1mSnZ2aGptdyAgIDEgICAwICAgICAgICAgIDEgICAgICAgICAgICAwICAgICAgMy44a2IgICAgICAgICAgMy44a2IKZ3JlZW4gIG9wZW4gICAub3BlbmRpc3Ryb19zZWN1cml0eSAgICAgIDVmWHlhbkZuU2tDUUQ2bjFKUW1KTlEgICAxICAgMCAgICAgICAgIDEwICAgICAgICAgICAgMCAgICAgNzcuNWtiICAgICAgICAgNzcuNWtiCg==" + } + ], + "connectionId": "0242acfffe13000a-0000000a-00000005-1eb087a9beb83f3e-a32794b4.0", + "numRequests": 1, + "numErrors": 0 +} diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/multiple_tuples.json b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/multiple_tuples.json new file mode 100644 index 000000000..d0f7eec8a --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/multiple_tuples.json @@ -0,0 +1,208 @@ +[ + { + "sourceRequest": { + "Request-URI": "/", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "User-Agent": "python-requests/2.32.3", + "Accept-Encoding": "gzip, deflate, zstd", + "Accept": "*/*", + "Connection": "keep-alive", + "Authorization": "Basic YWRtaW46YWRtaW4=", + "body": "" + }, + "sourceResponse": { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 374, + "content-type": "application/json; charset=UTF-8", + "content-length": "538", + "body": "ewogICJuYW1lIiA6ICIzY2MwNjhhZDU0ZWIiLAogICJjbHVzdGVyX25hbWUiIDogImRvY2tlci1jbHVzdGVyIiwKICAiY2x1c3Rlcl91dWlkIiA6ICJQSUFhd0R3U1FLU1JMdG9hWEMtb1dnIiwKICAidmVyc2lvbiIgOiB7CiAgICAibnVtYmVyIiA6ICI3LjEwLjIiLAogICAgImJ1aWxkX2ZsYXZvciIgOiAib3NzIiwKICAgICJidWlsZF90eXBlIiA6ICJkb2NrZXIiLAogICAgImJ1aWxkX2hhc2giIDogIjc0N2UxY2M3MWRlZjA3NzI1Mzg3OGE1OTE0M2MxZjc4NWFmYTkyYjkiLAogICAgImJ1aWxkX2RhdGUiIDogIjIwMjEtMDEtMTNUMDA6NDI6MTIuNDM1MzI2WiIsCiAgICAiYnVpbGRfc25hcHNob3QiIDogZmFsc2UsCiAgICAibHVjZW5lX3ZlcnNpb24iIDogIjguNy4wIiwKICAgICJtaW5pbXVtX3dpcmVfY29tcGF0aWJpbGl0eV92ZXJzaW9uIiA6ICI2LjguMCIsCiAgICAibWluaW11bV9pbmRleF9jb21wYXRpYmlsaXR5X3ZlcnNpb24iIDogIjYuMC4wLWJldGExIgogIH0sCiAgInRhZ2xpbmUiIDogIllvdSBLbm93LCBmb3IgU2VhcmNoIgp9Cg==" + }, + "targetRequest": { + "Request-URI": "/", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "User-Agent": "python-requests/2.32.3", + "Accept-Encoding": "gzip, deflate, zstd", + "Accept": "*/*", + "Connection": "keep-alive", + "Authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "body": "" + }, + "targetResponses": [ + { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 341, + "content-type": "application/json; charset=UTF-8", + "content-length": "568", + "body": "ewogICJuYW1lIiA6ICJlNmIzYTQzNzA3NjMiLAogICJjbHVzdGVyX25hbWUiIDogImRvY2tlci1jbHVzdGVyIiwKICAiY2x1c3Rlcl91dWlkIiA6ICI4OHpsaWs3T1JGdUZUNzh6ZHVyYWRBIiwKICAidmVyc2lvbiIgOiB7CiAgICAiZGlzdHJpYnV0aW9uIiA6ICJvcGVuc2VhcmNoIiwKICAgICJudW1iZXIiIDogIjIuMTUuMCIsCiAgICAiYnVpbGRfdHlwZSIgOiAidGFyIiwKICAgICJidWlsZF9oYXNoIiA6ICI2MWRiY2QwNzk1YzliZmU5YjgxZTU3NjIxNzU0MTRiYzM4YmJjYWRmIiwKICAgICJidWlsZF9kYXRlIiA6ICIyMDI0LTA2LTIwVDAzOjI3OjMyLjU2MjAzNjg5MFoiLAogICAgImJ1aWxkX3NuYXBzaG90IiA6IGZhbHNlLAogICAgImx1Y2VuZV92ZXJzaW9uIiA6ICI5LjEwLjAiLAogICAgIm1pbmltdW1fd2lyZV9jb21wYXRpYmlsaXR5X3ZlcnNpb24iIDogIjcuMTAuMCIsCiAgICAibWluaW11bV9pbmRleF9jb21wYXRpYmlsaXR5X3ZlcnNpb24iIDogIjcuMC4wIgogIH0sCiAgInRhZ2xpbmUiIDogIlRoZSBPcGVuU2VhcmNoIFByb2plY3Q6IGh0dHBzOi8vb3BlbnNlYXJjaC5vcmcvIgp9Cg==" + } + ], + "connectionId": "0242acfffe12000b-0000000a-00000003-4c7bb8f4aefa3189-c0544dbf.0", + "numRequests": 1, + "numErrors": 0 + }, + { + "sourceRequest": { + "Request-URI": "/geonames", + "Method": "PUT", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46YWRtaW4=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "1214", + "body": "eyJzZXR0aW5ncyI6eyJpbmRleC5udW1iZXJfb2Zfc2hhcmRzIjo1LCJpbmRleC5udW1iZXJfb2ZfcmVwbGljYXMiOjAsImluZGV4LnN0b3JlLnR5cGUiOiJmcyIsImluZGV4LnF1ZXJpZXMuY2FjaGUuZW5hYmxlZCI6ZmFsc2UsImluZGV4LnJlcXVlc3RzLmNhY2hlLmVuYWJsZSI6ZmFsc2V9LCJtYXBwaW5ncyI6eyJkeW5hbWljIjoic3RyaWN0IiwiX3NvdXJjZSI6eyJlbmFibGVkIjp0cnVlfSwicHJvcGVydGllcyI6eyJlbGV2YXRpb24iOnsidHlwZSI6ImludGVnZXIifSwibmFtZSI6eyJ0eXBlIjoidGV4dCIsImZpZWxkcyI6eyJyYXciOnsidHlwZSI6ImtleXdvcmQifX19LCJnZW9uYW1laWQiOnsidHlwZSI6ImxvbmcifSwiZmVhdHVyZV9jbGFzcyI6eyJ0eXBlIjoidGV4dCIsImZpZWxkcyI6eyJyYXciOnsidHlwZSI6ImtleXdvcmQifX19LCJsb2NhdGlvbiI6eyJ0eXBlIjoiZ2VvX3BvaW50In0sImNjMiI6eyJ0eXBlIjoidGV4dCIsImZpZWxkcyI6eyJyYXciOnsidHlwZSI6ImtleXdvcmQifX19LCJ0aW1lem9uZSI6eyJ0eXBlIjoidGV4dCIsImZpZWxkcyI6eyJyYXciOnsidHlwZSI6ImtleXdvcmQifX19LCJkZW0iOnsidHlwZSI6InRleHQiLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiY291bnRyeV9jb2RlIjp7InR5cGUiOiJ0ZXh0IiwiZmllbGRkYXRhIjp0cnVlLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiYWRtaW4xX2NvZGUiOnsidHlwZSI6InRleHQiLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiYWRtaW4yX2NvZGUiOnsidHlwZSI6InRleHQiLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiYWRtaW4zX2NvZGUiOnsidHlwZSI6InRleHQiLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiYWRtaW40X2NvZGUiOnsidHlwZSI6InRleHQiLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiZmVhdHVyZV9jb2RlIjp7InR5cGUiOiJ0ZXh0IiwiZmllbGRzIjp7InJhdyI6eyJ0eXBlIjoia2V5d29yZCJ9fX0sImFsdGVybmF0ZW5hbWVzIjp7InR5cGUiOiJ0ZXh0IiwiZmllbGRzIjp7InJhdyI6eyJ0eXBlIjoia2V5d29yZCJ9fX0sImFzY2lpbmFtZSI6eyJ0eXBlIjoidGV4dCIsImZpZWxkcyI6eyJyYXciOnsidHlwZSI6ImtleXdvcmQifX19LCJwb3B1bGF0aW9uIjp7InR5cGUiOiJsb25nIn19fX0=" + }, + "sourceResponse": { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 232, + "content-type": "application/json; charset=UTF-8", + "content-length": "67", + "body": "eyJhY2tub3dsZWRnZWQiOnRydWUsInNoYXJkc19hY2tub3dsZWRnZWQiOnRydWUsImluZGV4IjoiZ2VvbmFtZXMifQ==" + }, + "targetRequest": { + "Request-URI": "/geonames", + "Method": "PUT", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "1214", + "body": "eyJzZXR0aW5ncyI6eyJpbmRleC5udW1iZXJfb2Zfc2hhcmRzIjo1LCJpbmRleC5udW1iZXJfb2ZfcmVwbGljYXMiOjAsImluZGV4LnN0b3JlLnR5cGUiOiJmcyIsImluZGV4LnF1ZXJpZXMuY2FjaGUuZW5hYmxlZCI6ZmFsc2UsImluZGV4LnJlcXVlc3RzLmNhY2hlLmVuYWJsZSI6ZmFsc2V9LCJtYXBwaW5ncyI6eyJkeW5hbWljIjoic3RyaWN0IiwiX3NvdXJjZSI6eyJlbmFibGVkIjp0cnVlfSwicHJvcGVydGllcyI6eyJlbGV2YXRpb24iOnsidHlwZSI6ImludGVnZXIifSwibmFtZSI6eyJ0eXBlIjoidGV4dCIsImZpZWxkcyI6eyJyYXciOnsidHlwZSI6ImtleXdvcmQifX19LCJnZW9uYW1laWQiOnsidHlwZSI6ImxvbmcifSwiZmVhdHVyZV9jbGFzcyI6eyJ0eXBlIjoidGV4dCIsImZpZWxkcyI6eyJyYXciOnsidHlwZSI6ImtleXdvcmQifX19LCJsb2NhdGlvbiI6eyJ0eXBlIjoiZ2VvX3BvaW50In0sImNjMiI6eyJ0eXBlIjoidGV4dCIsImZpZWxkcyI6eyJyYXciOnsidHlwZSI6ImtleXdvcmQifX19LCJ0aW1lem9uZSI6eyJ0eXBlIjoidGV4dCIsImZpZWxkcyI6eyJyYXciOnsidHlwZSI6ImtleXdvcmQifX19LCJkZW0iOnsidHlwZSI6InRleHQiLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiY291bnRyeV9jb2RlIjp7InR5cGUiOiJ0ZXh0IiwiZmllbGRkYXRhIjp0cnVlLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiYWRtaW4xX2NvZGUiOnsidHlwZSI6InRleHQiLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiYWRtaW4yX2NvZGUiOnsidHlwZSI6InRleHQiLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiYWRtaW4zX2NvZGUiOnsidHlwZSI6InRleHQiLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiYWRtaW40X2NvZGUiOnsidHlwZSI6InRleHQiLCJmaWVsZHMiOnsicmF3Ijp7InR5cGUiOiJrZXl3b3JkIn19fSwiZmVhdHVyZV9jb2RlIjp7InR5cGUiOiJ0ZXh0IiwiZmllbGRzIjp7InJhdyI6eyJ0eXBlIjoia2V5d29yZCJ9fX0sImFsdGVybmF0ZW5hbWVzIjp7InR5cGUiOiJ0ZXh0IiwiZmllbGRzIjp7InJhdyI6eyJ0eXBlIjoia2V5d29yZCJ9fX0sImFzY2lpbmFtZSI6eyJ0eXBlIjoidGV4dCIsImZpZWxkcyI6eyJyYXciOnsidHlwZSI6ImtleXdvcmQifX19LCJwb3B1bGF0aW9uIjp7InR5cGUiOiJsb25nIn19fX0=" + }, + "targetResponses": [ + { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 270, + "content-type": "application/json; charset=UTF-8", + "content-length": "67", + "body": "eyJhY2tub3dsZWRnZWQiOnRydWUsInNoYXJkc19hY2tub3dsZWRnZWQiOnRydWUsImluZGV4IjoiZ2VvbmFtZXMifQ==" + } + ], + "connectionId": "0242acfffe12000b-0000000a-00000009-19ac20d3defa9804-07697cc6.0", + "numRequests": 1, + "numErrors": 0 + }, + { + "sourceRequest": { + "Request-URI": "/_cluster/health/geonames?wait_for_status=green&wait_for_no_relocating_shards=true", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46YWRtaW4=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "body": "" + }, + "sourceResponse": { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 3, + "content-type": "application/json; charset=UTF-8", + "content-length": "390", + "body": "eyJjbHVzdGVyX25hbWUiOiJkb2NrZXItY2x1c3RlciIsInN0YXR1cyI6ImdyZWVuIiwidGltZWRfb3V0IjpmYWxzZSwibnVtYmVyX29mX25vZGVzIjoxLCJudW1iZXJfb2ZfZGF0YV9ub2RlcyI6MSwiYWN0aXZlX3ByaW1hcnlfc2hhcmRzIjo1LCJhY3RpdmVfc2hhcmRzIjo1LCJyZWxvY2F0aW5nX3NoYXJkcyI6MCwiaW5pdGlhbGl6aW5nX3NoYXJkcyI6MCwidW5hc3NpZ25lZF9zaGFyZHMiOjAsImRlbGF5ZWRfdW5hc3NpZ25lZF9zaGFyZHMiOjAsIm51bWJlcl9vZl9wZW5kaW5nX3Rhc2tzIjowLCJudW1iZXJfb2ZfaW5fZmxpZ2h0X2ZldGNoIjowLCJ0YXNrX21heF93YWl0aW5nX2luX3F1ZXVlX21pbGxpcyI6MCwiYWN0aXZlX3NoYXJkc19wZXJjZW50X2FzX251bWJlciI6MTAwLjB9" + }, + "targetRequest": { + "Request-URI": "/_cluster/health/geonames?wait_for_status=green&wait_for_no_relocating_shards=true", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "body": "" + }, + "targetResponses": [ + { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 29, + "content-type": "application/json; charset=UTF-8", + "content-length": "449", + "body": "eyJjbHVzdGVyX25hbWUiOiJkb2NrZXItY2x1c3RlciIsInN0YXR1cyI6ImdyZWVuIiwidGltZWRfb3V0IjpmYWxzZSwibnVtYmVyX29mX25vZGVzIjoxLCJudW1iZXJfb2ZfZGF0YV9ub2RlcyI6MSwiZGlzY292ZXJlZF9tYXN0ZXIiOnRydWUsImRpc2NvdmVyZWRfY2x1c3Rlcl9tYW5hZ2VyIjp0cnVlLCJhY3RpdmVfcHJpbWFyeV9zaGFyZHMiOjUsImFjdGl2ZV9zaGFyZHMiOjUsInJlbG9jYXRpbmdfc2hhcmRzIjowLCJpbml0aWFsaXppbmdfc2hhcmRzIjowLCJ1bmFzc2lnbmVkX3NoYXJkcyI6MCwiZGVsYXllZF91bmFzc2lnbmVkX3NoYXJkcyI6MCwibnVtYmVyX29mX3BlbmRpbmdfdGFza3MiOjAsIm51bWJlcl9vZl9pbl9mbGlnaHRfZmV0Y2giOjAsInRhc2tfbWF4X3dhaXRpbmdfaW5fcXVldWVfbWlsbGlzIjowLCJhY3RpdmVfc2hhcmRzX3BlcmNlbnRfYXNfbnVtYmVyIjoxMDAuMH0=" + } + ], + "connectionId": "0242acfffe12000b-0000000a-0000000b-2681c86bdefa99ec-45c6b416.0", + "numRequests": 1, + "numErrors": 0 + }, + { + "sourceRequest": { + "Request-URI": "/_bulk", + "Method": "POST", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46YWRtaW4=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "3974", + "body": "eyJpbmRleCI6IHsiX2luZGV4IjogImdlb25hbWVzIn19CnsiZ2VvbmFtZWlkIjogMjk4NjA0MywgIm5hbWUiOiAiUGljIGRlIEZvbnQgQmxhbmNhIiwgImFzY2lpbmFtZSI6ICJQaWMgZGUgRm9udCBCbGFuY2EiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUGljIGRlIEZvbnQgQmxhbmNhLFBpYyBkdSBQb3J0IiwgImZlYXR1cmVfY2xhc3MiOiAiVCIsICJmZWF0dXJlX2NvZGUiOiAiUEsiLCAiY291bnRyeV9jb2RlIjogIkFEIiwgImFkbWluMV9jb2RlIjogIjAwIiwgInBvcHVsYXRpb24iOiAwLCAiZGVtIjogIjI4NjAiLCAidGltZXpvbmUiOiAiRXVyb3BlL0FuZG9ycmEiLCAibG9jYXRpb24iOiBbMS41MzMzNSwgNDIuNjQ5OTFdfQp7ImluZGV4IjogeyJfaW5kZXgiOiAiZ2VvbmFtZXMifX0KeyJnZW9uYW1laWQiOiAyOTkzODM4LCAibmFtZSI6ICJQaWMgZGUgTWlsLU1lbnV0IiwgImFzY2lpbmFtZSI6ICJQaWMgZGUgTWlsLU1lbnV0IiwgImFsdGVybmF0ZW5hbWVzIjogIlBpYyBkZSBNaWwtTWVudXQiLCAiZmVhdHVyZV9jbGFzcyI6ICJUIiwgImZlYXR1cmVfY29kZSI6ICJQSyIsICJjb3VudHJ5X2NvZGUiOiAiQUQiLCAiY2MyIjogIkFELEZSIiwgImFkbWluMV9jb2RlIjogIkIzIiwgImFkbWluMl9jb2RlIjogIjA5IiwgImFkbWluM19jb2RlIjogIjA5MSIsICJhZG1pbjRfY29kZSI6ICIwOTAyNCIsICJwb3B1bGF0aW9uIjogMCwgImRlbSI6ICIyMTM4IiwgInRpbWV6b25lIjogIkV1cm9wZS9BbmRvcnJhIiwgImxvY2F0aW9uIjogWzEuNjUsIDQyLjYzMzMzXX0KeyJpbmRleCI6IHsiX2luZGV4IjogImdlb25hbWVzIn19CnsiZ2VvbmFtZWlkIjogMjk5NDcwMSwgIm5hbWUiOiAiUm9jIE3DqWzDqSIsICJhc2NpaW5hbWUiOiAiUm9jIE1lbGUiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUm9jIE1lbGUsUm9jIE1lbGVyLFJvYyBNw6lsw6kiLCAiZmVhdHVyZV9jbGFzcyI6ICJUIiwgImZlYXR1cmVfY29kZSI6ICJNVCIsICJjb3VudHJ5X2NvZGUiOiAiQUQiLCAiY2MyIjogIkFELEZSIiwgImFkbWluMV9jb2RlIjogIjAwIiwgInBvcHVsYXRpb24iOiAwLCAiZGVtIjogIjI4MDMiLCAidGltZXpvbmUiOiAiRXVyb3BlL0FuZG9ycmEiLCAibG9jYXRpb24iOiBbMS43NDAyOCwgNDIuNTg3NjVdfQp7ImluZGV4IjogeyJfaW5kZXgiOiAiZ2VvbmFtZXMifX0KeyJnZW9uYW1laWQiOiAzMDA3NjgzLCAibmFtZSI6ICJQaWMgZGVzIExhbmdvdW5lbGxlcyIsICJhc2NpaW5hbWUiOiAiUGljIGRlcyBMYW5nb3VuZWxsZXMiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUGljIGRlcyBMYW5nb3VuZWxsZXMiLCAiZmVhdHVyZV9jbGFzcyI6ICJUIiwgImZlYXR1cmVfY29kZSI6ICJQSyIsICJjb3VudHJ5X2NvZGUiOiAiQUQiLCAiY2MyIjogIkFELEZSIiwgImFkbWluMV9jb2RlIjogIjAwIiwgInBvcHVsYXRpb24iOiAwLCAiZGVtIjogIjI2ODUiLCAidGltZXpvbmUiOiAiRXVyb3BlL0FuZG9ycmEiLCAibG9jYXRpb24iOiBbMS40NzM2NCwgNDIuNjEyMDNdfQp7ImluZGV4IjogeyJfaW5kZXgiOiAiZ2VvbmFtZXMifX0KeyJnZW9uYW1laWQiOiAzMDE3ODMyLCAibmFtZSI6ICJQaWMgZGUgbGVzIEFiZWxsZXRlcyIsICJhc2NpaW5hbWUiOiAiUGljIGRlIGxlcyBBYmVsbGV0ZXMiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUGljIGRlIGxhIEZvbnQtTmVncmUsUGljIGRlIGxhIEZvbnQtTsOoZ3JlLFBpYyBkZSBsZXMgQWJlbGxldGVzIiwgImZlYXR1cmVfY2xhc3MiOiAiVCIsICJmZWF0dXJlX2NvZGUiOiAiUEsiLCAiY291bnRyeV9jb2RlIjogIkFEIiwgImNjMiI6ICJGUiIsICJhZG1pbjFfY29kZSI6ICJBOSIsICJhZG1pbjJfY29kZSI6ICI2NiIsICJhZG1pbjNfY29kZSI6ICI2NjMiLCAiYWRtaW40X2NvZGUiOiAiNjYxNDYiLCAicG9wdWxhdGlvbiI6IDAsICJkZW0iOiAiMjQxMSIsICJ0aW1lem9uZSI6ICJFdXJvcGUvQW5kb3JyYSIsICJsb2NhdGlvbiI6IFsxLjczMzQzLCA0Mi41MjUzNV19CnsiaW5kZXgiOiB7Il9pbmRleCI6ICJnZW9uYW1lcyJ9fQp7Imdlb25hbWVpZCI6IDMwMTc4MzMsICJuYW1lIjogIkVzdGFueSBkZSBsZXMgQWJlbGxldGVzIiwgImFzY2lpbmFtZSI6ICJFc3RhbnkgZGUgbGVzIEFiZWxsZXRlcyIsICJhbHRlcm5hdGVuYW1lcyI6ICJFc3RhbnkgZGUgbGVzIEFiZWxsZXRlcyxFdGFuZyBkZSBGb250LU5lZ3JlLMOJdGFuZyBkZSBGb250LU7DqGdyZSIsICJmZWF0dXJlX2NsYXNzIjogIkgiLCAiZmVhdHVyZV9jb2RlIjogIkxLIiwgImNvdW50cnlfY29kZSI6ICJBRCIsICJjYzIiOiAiRlIiLCAiYWRtaW4xX2NvZGUiOiAiQTkiLCAicG9wdWxhdGlvbiI6IDAsICJkZW0iOiAiMjI2MCIsICJ0aW1lem9uZSI6ICJFdXJvcGUvQW5kb3JyYSIsICJsb2NhdGlvbiI6IFsxLjczMzYyLCA0Mi41MjkxNV19CnsiaW5kZXgiOiB7Il9pbmRleCI6ICJnZW9uYW1lcyJ9fQp7Imdlb25hbWVpZCI6IDMwMjMyMDMsICJuYW1lIjogIlBvcnQgVmlldXggZGUgbGEgQ291bWUgZOKAmU9zZSIsICJhc2NpaW5hbWUiOiAiUG9ydCBWaWV1eCBkZSBsYSBDb3VtZSBkJ09zZSIsICJhbHRlcm5hdGVuYW1lcyI6ICJQb3J0IFZpZXV4IGRlIENvdW1lIGQnT3NlLFBvcnQgVmlldXggZGUgQ291bWUgZOKAmU9zZSxQb3J0IFZpZXV4IGRlIGxhIENvdW1lIGQnT3NlLFBvcnQgVmlldXggZGUgbGEgQ291bWUgZOKAmU9zZSIsICJmZWF0dXJlX2NsYXNzIjogIlQiLCAiZmVhdHVyZV9jb2RlIjogIlBBU1MiLCAiY291bnRyeV9jb2RlIjogIkFEIiwgImFkbWluMV9jb2RlIjogIjAwIiwgInBvcHVsYXRpb24iOiAwLCAiZGVtIjogIjI2ODciLCAidGltZXpvbmUiOiAiRXVyb3BlL0FuZG9ycmEiLCAibG9jYXRpb24iOiBbMS42MTgyMywgNDIuNjI1NjhdfQp7ImluZGV4IjogeyJfaW5kZXgiOiAiZ2VvbmFtZXMifX0KeyJnZW9uYW1laWQiOiAzMDI5MzE1LCAibmFtZSI6ICJQb3J0IGRlIGxhIENhYmFuZXR0ZSIsICJhc2NpaW5hbWUiOiAiUG9ydCBkZSBsYSBDYWJhbmV0dGUiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUG9ydCBkZSBsYSBDYWJhbmV0dGUsUG9ydGVpbGxlIGRlIGxhIENhYmFuZXR0ZSIsICJmZWF0dXJlX2NsYXNzIjogIlQiLCAiZmVhdHVyZV9jb2RlIjogIlBBU1MiLCAiY291bnRyeV9jb2RlIjogIkFEIiwgImNjMiI6ICJBRCxGUiIsICJhZG1pbjFfY29kZSI6ICJCMyIsICJhZG1pbjJfY29kZSI6ICIwOSIsICJhZG1pbjNfY29kZSI6ICIwOTEiLCAiYWRtaW40X2NvZGUiOiAiMDkxMzkiLCAicG9wdWxhdGlvbiI6IDAsICJkZW0iOiAiMjM3OSIsICJ0aW1lem9uZSI6ICJFdXJvcGUvQW5kb3JyYSIsICJsb2NhdGlvbiI6IFsxLjczMzMzLCA0Mi42XX0KeyJpbmRleCI6IHsiX2luZGV4IjogImdlb25hbWVzIn19CnsiZ2VvbmFtZWlkIjogMzAzNDk0NSwgIm5hbWUiOiAiUG9ydCBEcmV0IiwgImFzY2lpbmFtZSI6ICJQb3J0IERyZXQiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUG9ydCBEcmV0LFBvcnQgZGUgQmFyZWl0ZXMsUG9ydCBkZSBsYXMgQmFyZXl0ZXMsUG9ydCBkZXMgQmFyZXl0ZXMiLCAiZmVhdHVyZV9jbGFzcyI6ICJUIiwgImZlYXR1cmVfY29kZSI6ICJQQVNTIiwgImNvdW50cnlfY29kZSI6ICJBRCIsICJhZG1pbjFfY29kZSI6ICIwMCIsICJwb3B1bGF0aW9uIjogMCwgImRlbSI6ICIyNjYwIiwgInRpbWV6b25lIjogIkV1cm9wZS9BbmRvcnJhIiwgImxvY2F0aW9uIjogWzEuNDU1NjIsIDQyLjYwMTcyXX0KeyJpbmRleCI6IHsiX2luZGV4IjogImdlb25hbWVzIn19CnsiZ2VvbmFtZWlkIjogMzAzODgxNCwgIm5hbWUiOiAiQ29zdGEgZGUgWHVyaXVzIiwgImFzY2lpbmFtZSI6ICJDb3N0YSBkZSBYdXJpdXMiLCAiZmVhdHVyZV9jbGFzcyI6ICJUIiwgImZlYXR1cmVfY29kZSI6ICJTTFAiLCAiY291bnRyeV9jb2RlIjogIkFEIiwgImFkbWluMV9jb2RlIjogIjA3IiwgInBvcHVsYXRpb24iOiAwLCAiZGVtIjogIjE4MzkiLCAidGltZXpvbmUiOiAiRXVyb3BlL0FuZG9ycmEiLCAibG9jYXRpb24iOiBbMS40NzU2OSwgNDIuNTA2OTJdfQo=" + }, + "sourceResponse": { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 47, + "content-type": "application/json; charset=UTF-8", + "content-length": "2026", + "body": "eyJ0b29rIjozNSwiZXJyb3JzIjpmYWxzZSwiaXRlbXMiOlt7ImluZGV4Ijp7Il9pbmRleCI6Imdlb25hbWVzIiwiX3R5cGUiOiJfZG9jIiwiX2lkIjoiSTlENkM1SUJIQ0R0OTNrLW5CSXciLCJfdmVyc2lvbiI6MSwicmVzdWx0IjoiY3JlYXRlZCIsIl9zaGFyZHMiOnsidG90YWwiOjEsInN1Y2Nlc3NmdWwiOjEsImZhaWxlZCI6MH0sIl9zZXFfbm8iOjAsIl9wcmltYXJ5X3Rlcm0iOjEsInN0YXR1cyI6MjAxfX0seyJpbmRleCI6eyJfaW5kZXgiOiJnZW9uYW1lcyIsIl90eXBlIjoiX2RvYyIsIl9pZCI6IkpORDZDNUlCSENEdDkzay1uQkl4IiwiX3ZlcnNpb24iOjEsInJlc3VsdCI6ImNyZWF0ZWQiLCJfc2hhcmRzIjp7InRvdGFsIjoxLCJzdWNjZXNzZnVsIjoxLCJmYWlsZWQiOjB9LCJfc2VxX25vIjowLCJfcHJpbWFyeV90ZXJtIjoxLCJzdGF0dXMiOjIwMX19LHsiaW5kZXgiOnsiX2luZGV4IjoiZ2VvbmFtZXMiLCJfdHlwZSI6Il9kb2MiLCJfaWQiOiJKZEQ2QzVJQkhDRHQ5M2stbkJJeCIsIl92ZXJzaW9uIjoxLCJyZXN1bHQiOiJjcmVhdGVkIiwiX3NoYXJkcyI6eyJ0b3RhbCI6MSwic3VjY2Vzc2Z1bCI6MSwiZmFpbGVkIjowfSwiX3NlcV9ubyI6MSwiX3ByaW1hcnlfdGVybSI6MSwic3RhdHVzIjoyMDF9fSx7ImluZGV4Ijp7Il9pbmRleCI6Imdlb25hbWVzIiwiX3R5cGUiOiJfZG9jIiwiX2lkIjoiSnRENkM1SUJIQ0R0OTNrLW5CSXgiLCJfdmVyc2lvbiI6MSwicmVzdWx0IjoiY3JlYXRlZCIsIl9zaGFyZHMiOnsidG90YWwiOjEsInN1Y2Nlc3NmdWwiOjEsImZhaWxlZCI6MH0sIl9zZXFfbm8iOjAsIl9wcmltYXJ5X3Rlcm0iOjEsInN0YXR1cyI6MjAxfX0seyJpbmRleCI6eyJfaW5kZXgiOiJnZW9uYW1lcyIsIl90eXBlIjoiX2RvYyIsIl9pZCI6Iko5RDZDNUlCSENEdDkzay1uQkl4IiwiX3ZlcnNpb24iOjEsInJlc3VsdCI6ImNyZWF0ZWQiLCJfc2hhcmRzIjp7InRvdGFsIjoxLCJzdWNjZXNzZnVsIjoxLCJmYWlsZWQiOjB9LCJfc2VxX25vIjowLCJfcHJpbWFyeV90ZXJtIjoxLCJzdGF0dXMiOjIwMX19LHsiaW5kZXgiOnsiX2luZGV4IjoiZ2VvbmFtZXMiLCJfdHlwZSI6Il9kb2MiLCJfaWQiOiJLTkQ2QzVJQkhDRHQ5M2stbkJJeCIsIl92ZXJzaW9uIjoxLCJyZXN1bHQiOiJjcmVhdGVkIiwiX3NoYXJkcyI6eyJ0b3RhbCI6MSwic3VjY2Vzc2Z1bCI6MSwiZmFpbGVkIjowfSwiX3NlcV9ubyI6MiwiX3ByaW1hcnlfdGVybSI6MSwic3RhdHVzIjoyMDF9fSx7ImluZGV4Ijp7Il9pbmRleCI6Imdlb25hbWVzIiwiX3R5cGUiOiJfZG9jIiwiX2lkIjoiS2RENkM1SUJIQ0R0OTNrLW5CSXgiLCJfdmVyc2lvbiI6MSwicmVzdWx0IjoiY3JlYXRlZCIsIl9zaGFyZHMiOnsidG90YWwiOjEsInN1Y2Nlc3NmdWwiOjEsImZhaWxlZCI6MH0sIl9zZXFfbm8iOjEsIl9wcmltYXJ5X3Rlcm0iOjEsInN0YXR1cyI6MjAxfX0seyJpbmRleCI6eyJfaW5kZXgiOiJnZW9uYW1lcyIsIl90eXBlIjoiX2RvYyIsIl9pZCI6Ikt0RDZDNUlCSENEdDkzay1uQkl4IiwiX3ZlcnNpb24iOjEsInJlc3VsdCI6ImNyZWF0ZWQiLCJfc2hhcmRzIjp7InRvdGFsIjoxLCJzdWNjZXNzZnVsIjoxLCJmYWlsZWQiOjB9LCJfc2VxX25vIjozLCJfcHJpbWFyeV90ZXJtIjoxLCJzdGF0dXMiOjIwMX19LHsiaW5kZXgiOnsiX2luZGV4IjoiZ2VvbmFtZXMiLCJfdHlwZSI6Il9kb2MiLCJfaWQiOiJLOUQ2QzVJQkhDRHQ5M2stbkJJeCIsIl92ZXJzaW9uIjoxLCJyZXN1bHQiOiJjcmVhdGVkIiwiX3NoYXJkcyI6eyJ0b3RhbCI6MSwic3VjY2Vzc2Z1bCI6MSwiZmFpbGVkIjowfSwiX3NlcV9ubyI6MSwiX3ByaW1hcnlfdGVybSI6MSwic3RhdHVzIjoyMDF9fSx7ImluZGV4Ijp7Il9pbmRleCI6Imdlb25hbWVzIiwiX3R5cGUiOiJfZG9jIiwiX2lkIjoiTE5ENkM1SUJIQ0R0OTNrLW5CSXgiLCJfdmVyc2lvbiI6MSwicmVzdWx0IjoiY3JlYXRlZCIsIl9zaGFyZHMiOnsidG90YWwiOjEsInN1Y2Nlc3NmdWwiOjEsImZhaWxlZCI6MH0sIl9zZXFfbm8iOjAsIl9wcmltYXJ5X3Rlcm0iOjEsInN0YXR1cyI6MjAxfX1dfQ==" + }, + "targetRequest": { + "Request-URI": "/_bulk", + "Method": "POST", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "3974", + "body": "eyJpbmRleCI6IHsiX2luZGV4IjogImdlb25hbWVzIn19CnsiZ2VvbmFtZWlkIjogMjk4NjA0MywgIm5hbWUiOiAiUGljIGRlIEZvbnQgQmxhbmNhIiwgImFzY2lpbmFtZSI6ICJQaWMgZGUgRm9udCBCbGFuY2EiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUGljIGRlIEZvbnQgQmxhbmNhLFBpYyBkdSBQb3J0IiwgImZlYXR1cmVfY2xhc3MiOiAiVCIsICJmZWF0dXJlX2NvZGUiOiAiUEsiLCAiY291bnRyeV9jb2RlIjogIkFEIiwgImFkbWluMV9jb2RlIjogIjAwIiwgInBvcHVsYXRpb24iOiAwLCAiZGVtIjogIjI4NjAiLCAidGltZXpvbmUiOiAiRXVyb3BlL0FuZG9ycmEiLCAibG9jYXRpb24iOiBbMS41MzMzNSwgNDIuNjQ5OTFdfQp7ImluZGV4IjogeyJfaW5kZXgiOiAiZ2VvbmFtZXMifX0KeyJnZW9uYW1laWQiOiAyOTkzODM4LCAibmFtZSI6ICJQaWMgZGUgTWlsLU1lbnV0IiwgImFzY2lpbmFtZSI6ICJQaWMgZGUgTWlsLU1lbnV0IiwgImFsdGVybmF0ZW5hbWVzIjogIlBpYyBkZSBNaWwtTWVudXQiLCAiZmVhdHVyZV9jbGFzcyI6ICJUIiwgImZlYXR1cmVfY29kZSI6ICJQSyIsICJjb3VudHJ5X2NvZGUiOiAiQUQiLCAiY2MyIjogIkFELEZSIiwgImFkbWluMV9jb2RlIjogIkIzIiwgImFkbWluMl9jb2RlIjogIjA5IiwgImFkbWluM19jb2RlIjogIjA5MSIsICJhZG1pbjRfY29kZSI6ICIwOTAyNCIsICJwb3B1bGF0aW9uIjogMCwgImRlbSI6ICIyMTM4IiwgInRpbWV6b25lIjogIkV1cm9wZS9BbmRvcnJhIiwgImxvY2F0aW9uIjogWzEuNjUsIDQyLjYzMzMzXX0KeyJpbmRleCI6IHsiX2luZGV4IjogImdlb25hbWVzIn19CnsiZ2VvbmFtZWlkIjogMjk5NDcwMSwgIm5hbWUiOiAiUm9jIE3DqWzDqSIsICJhc2NpaW5hbWUiOiAiUm9jIE1lbGUiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUm9jIE1lbGUsUm9jIE1lbGVyLFJvYyBNw6lsw6kiLCAiZmVhdHVyZV9jbGFzcyI6ICJUIiwgImZlYXR1cmVfY29kZSI6ICJNVCIsICJjb3VudHJ5X2NvZGUiOiAiQUQiLCAiY2MyIjogIkFELEZSIiwgImFkbWluMV9jb2RlIjogIjAwIiwgInBvcHVsYXRpb24iOiAwLCAiZGVtIjogIjI4MDMiLCAidGltZXpvbmUiOiAiRXVyb3BlL0FuZG9ycmEiLCAibG9jYXRpb24iOiBbMS43NDAyOCwgNDIuNTg3NjVdfQp7ImluZGV4IjogeyJfaW5kZXgiOiAiZ2VvbmFtZXMifX0KeyJnZW9uYW1laWQiOiAzMDA3NjgzLCAibmFtZSI6ICJQaWMgZGVzIExhbmdvdW5lbGxlcyIsICJhc2NpaW5hbWUiOiAiUGljIGRlcyBMYW5nb3VuZWxsZXMiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUGljIGRlcyBMYW5nb3VuZWxsZXMiLCAiZmVhdHVyZV9jbGFzcyI6ICJUIiwgImZlYXR1cmVfY29kZSI6ICJQSyIsICJjb3VudHJ5X2NvZGUiOiAiQUQiLCAiY2MyIjogIkFELEZSIiwgImFkbWluMV9jb2RlIjogIjAwIiwgInBvcHVsYXRpb24iOiAwLCAiZGVtIjogIjI2ODUiLCAidGltZXpvbmUiOiAiRXVyb3BlL0FuZG9ycmEiLCAibG9jYXRpb24iOiBbMS40NzM2NCwgNDIuNjEyMDNdfQp7ImluZGV4IjogeyJfaW5kZXgiOiAiZ2VvbmFtZXMifX0KeyJnZW9uYW1laWQiOiAzMDE3ODMyLCAibmFtZSI6ICJQaWMgZGUgbGVzIEFiZWxsZXRlcyIsICJhc2NpaW5hbWUiOiAiUGljIGRlIGxlcyBBYmVsbGV0ZXMiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUGljIGRlIGxhIEZvbnQtTmVncmUsUGljIGRlIGxhIEZvbnQtTsOoZ3JlLFBpYyBkZSBsZXMgQWJlbGxldGVzIiwgImZlYXR1cmVfY2xhc3MiOiAiVCIsICJmZWF0dXJlX2NvZGUiOiAiUEsiLCAiY291bnRyeV9jb2RlIjogIkFEIiwgImNjMiI6ICJGUiIsICJhZG1pbjFfY29kZSI6ICJBOSIsICJhZG1pbjJfY29kZSI6ICI2NiIsICJhZG1pbjNfY29kZSI6ICI2NjMiLCAiYWRtaW40X2NvZGUiOiAiNjYxNDYiLCAicG9wdWxhdGlvbiI6IDAsICJkZW0iOiAiMjQxMSIsICJ0aW1lem9uZSI6ICJFdXJvcGUvQW5kb3JyYSIsICJsb2NhdGlvbiI6IFsxLjczMzQzLCA0Mi41MjUzNV19CnsiaW5kZXgiOiB7Il9pbmRleCI6ICJnZW9uYW1lcyJ9fQp7Imdlb25hbWVpZCI6IDMwMTc4MzMsICJuYW1lIjogIkVzdGFueSBkZSBsZXMgQWJlbGxldGVzIiwgImFzY2lpbmFtZSI6ICJFc3RhbnkgZGUgbGVzIEFiZWxsZXRlcyIsICJhbHRlcm5hdGVuYW1lcyI6ICJFc3RhbnkgZGUgbGVzIEFiZWxsZXRlcyxFdGFuZyBkZSBGb250LU5lZ3JlLMOJdGFuZyBkZSBGb250LU7DqGdyZSIsICJmZWF0dXJlX2NsYXNzIjogIkgiLCAiZmVhdHVyZV9jb2RlIjogIkxLIiwgImNvdW50cnlfY29kZSI6ICJBRCIsICJjYzIiOiAiRlIiLCAiYWRtaW4xX2NvZGUiOiAiQTkiLCAicG9wdWxhdGlvbiI6IDAsICJkZW0iOiAiMjI2MCIsICJ0aW1lem9uZSI6ICJFdXJvcGUvQW5kb3JyYSIsICJsb2NhdGlvbiI6IFsxLjczMzYyLCA0Mi41MjkxNV19CnsiaW5kZXgiOiB7Il9pbmRleCI6ICJnZW9uYW1lcyJ9fQp7Imdlb25hbWVpZCI6IDMwMjMyMDMsICJuYW1lIjogIlBvcnQgVmlldXggZGUgbGEgQ291bWUgZOKAmU9zZSIsICJhc2NpaW5hbWUiOiAiUG9ydCBWaWV1eCBkZSBsYSBDb3VtZSBkJ09zZSIsICJhbHRlcm5hdGVuYW1lcyI6ICJQb3J0IFZpZXV4IGRlIENvdW1lIGQnT3NlLFBvcnQgVmlldXggZGUgQ291bWUgZOKAmU9zZSxQb3J0IFZpZXV4IGRlIGxhIENvdW1lIGQnT3NlLFBvcnQgVmlldXggZGUgbGEgQ291bWUgZOKAmU9zZSIsICJmZWF0dXJlX2NsYXNzIjogIlQiLCAiZmVhdHVyZV9jb2RlIjogIlBBU1MiLCAiY291bnRyeV9jb2RlIjogIkFEIiwgImFkbWluMV9jb2RlIjogIjAwIiwgInBvcHVsYXRpb24iOiAwLCAiZGVtIjogIjI2ODciLCAidGltZXpvbmUiOiAiRXVyb3BlL0FuZG9ycmEiLCAibG9jYXRpb24iOiBbMS42MTgyMywgNDIuNjI1NjhdfQp7ImluZGV4IjogeyJfaW5kZXgiOiAiZ2VvbmFtZXMifX0KeyJnZW9uYW1laWQiOiAzMDI5MzE1LCAibmFtZSI6ICJQb3J0IGRlIGxhIENhYmFuZXR0ZSIsICJhc2NpaW5hbWUiOiAiUG9ydCBkZSBsYSBDYWJhbmV0dGUiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUG9ydCBkZSBsYSBDYWJhbmV0dGUsUG9ydGVpbGxlIGRlIGxhIENhYmFuZXR0ZSIsICJmZWF0dXJlX2NsYXNzIjogIlQiLCAiZmVhdHVyZV9jb2RlIjogIlBBU1MiLCAiY291bnRyeV9jb2RlIjogIkFEIiwgImNjMiI6ICJBRCxGUiIsICJhZG1pbjFfY29kZSI6ICJCMyIsICJhZG1pbjJfY29kZSI6ICIwOSIsICJhZG1pbjNfY29kZSI6ICIwOTEiLCAiYWRtaW40X2NvZGUiOiAiMDkxMzkiLCAicG9wdWxhdGlvbiI6IDAsICJkZW0iOiAiMjM3OSIsICJ0aW1lem9uZSI6ICJFdXJvcGUvQW5kb3JyYSIsICJsb2NhdGlvbiI6IFsxLjczMzMzLCA0Mi42XX0KeyJpbmRleCI6IHsiX2luZGV4IjogImdlb25hbWVzIn19CnsiZ2VvbmFtZWlkIjogMzAzNDk0NSwgIm5hbWUiOiAiUG9ydCBEcmV0IiwgImFzY2lpbmFtZSI6ICJQb3J0IERyZXQiLCAiYWx0ZXJuYXRlbmFtZXMiOiAiUG9ydCBEcmV0LFBvcnQgZGUgQmFyZWl0ZXMsUG9ydCBkZSBsYXMgQmFyZXl0ZXMsUG9ydCBkZXMgQmFyZXl0ZXMiLCAiZmVhdHVyZV9jbGFzcyI6ICJUIiwgImZlYXR1cmVfY29kZSI6ICJQQVNTIiwgImNvdW50cnlfY29kZSI6ICJBRCIsICJhZG1pbjFfY29kZSI6ICIwMCIsICJwb3B1bGF0aW9uIjogMCwgImRlbSI6ICIyNjYwIiwgInRpbWV6b25lIjogIkV1cm9wZS9BbmRvcnJhIiwgImxvY2F0aW9uIjogWzEuNDU1NjIsIDQyLjYwMTcyXX0KeyJpbmRleCI6IHsiX2luZGV4IjogImdlb25hbWVzIn19CnsiZ2VvbmFtZWlkIjogMzAzODgxNCwgIm5hbWUiOiAiQ29zdGEgZGUgWHVyaXVzIiwgImFzY2lpbmFtZSI6ICJDb3N0YSBkZSBYdXJpdXMiLCAiZmVhdHVyZV9jbGFzcyI6ICJUIiwgImZlYXR1cmVfY29kZSI6ICJTTFAiLCAiY291bnRyeV9jb2RlIjogIkFEIiwgImFkbWluMV9jb2RlIjogIjA3IiwgInBvcHVsYXRpb24iOiAwLCAiZGVtIjogIjE4MzkiLCAidGltZXpvbmUiOiAiRXVyb3BlL0FuZG9ycmEiLCAibG9jYXRpb24iOiBbMS40NzU2OSwgNDIuNTA2OTJdfQo=" + }, + "targetResponses": [ + { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 49, + "content-type": "application/json; charset=UTF-8", + "content-length": "1876", + "body": "eyJ0b29rIjoyNSwiZXJyb3JzIjpmYWxzZSwiaXRlbXMiOlt7ImluZGV4Ijp7Il9pbmRleCI6Imdlb25hbWVzIiwiX2lkIjoiMFlfNkM1SUIzdGszODYtVG5NOUYiLCJfdmVyc2lvbiI6MSwicmVzdWx0IjoiY3JlYXRlZCIsIl9zaGFyZHMiOnsidG90YWwiOjEsInN1Y2Nlc3NmdWwiOjEsImZhaWxlZCI6MH0sIl9zZXFfbm8iOjAsIl9wcmltYXJ5X3Rlcm0iOjEsInN0YXR1cyI6MjAxfX0seyJpbmRleCI6eyJfaW5kZXgiOiJnZW9uYW1lcyIsIl9pZCI6IjBvXzZDNUlCM3RrMzg2LVRuTTlGIiwiX3ZlcnNpb24iOjEsInJlc3VsdCI6ImNyZWF0ZWQiLCJfc2hhcmRzIjp7InRvdGFsIjoxLCJzdWNjZXNzZnVsIjoxLCJmYWlsZWQiOjB9LCJfc2VxX25vIjowLCJfcHJpbWFyeV90ZXJtIjoxLCJzdGF0dXMiOjIwMX19LHsiaW5kZXgiOnsiX2luZGV4IjoiZ2VvbmFtZXMiLCJfaWQiOiIwNF82QzVJQjN0azM4Ni1Ubk05RiIsIl92ZXJzaW9uIjoxLCJyZXN1bHQiOiJjcmVhdGVkIiwiX3NoYXJkcyI6eyJ0b3RhbCI6MSwic3VjY2Vzc2Z1bCI6MSwiZmFpbGVkIjowfSwiX3NlcV9ubyI6MSwiX3ByaW1hcnlfdGVybSI6MSwic3RhdHVzIjoyMDF9fSx7ImluZGV4Ijp7Il9pbmRleCI6Imdlb25hbWVzIiwiX2lkIjoiMUlfNkM1SUIzdGszODYtVG5NOUYiLCJfdmVyc2lvbiI6MSwicmVzdWx0IjoiY3JlYXRlZCIsIl9zaGFyZHMiOnsidG90YWwiOjEsInN1Y2Nlc3NmdWwiOjEsImZhaWxlZCI6MH0sIl9zZXFfbm8iOjIsIl9wcmltYXJ5X3Rlcm0iOjEsInN0YXR1cyI6MjAxfX0seyJpbmRleCI6eyJfaW5kZXgiOiJnZW9uYW1lcyIsIl9pZCI6IjFZXzZDNUlCM3RrMzg2LVRuTTlGIiwiX3ZlcnNpb24iOjEsInJlc3VsdCI6ImNyZWF0ZWQiLCJfc2hhcmRzIjp7InRvdGFsIjoxLCJzdWNjZXNzZnVsIjoxLCJmYWlsZWQiOjB9LCJfc2VxX25vIjowLCJfcHJpbWFyeV90ZXJtIjoxLCJzdGF0dXMiOjIwMX19LHsiaW5kZXgiOnsiX2luZGV4IjoiZ2VvbmFtZXMiLCJfaWQiOiIxb182QzVJQjN0azM4Ni1Ubk05RiIsIl92ZXJzaW9uIjoxLCJyZXN1bHQiOiJjcmVhdGVkIiwiX3NoYXJkcyI6eyJ0b3RhbCI6MSwic3VjY2Vzc2Z1bCI6MSwiZmFpbGVkIjowfSwiX3NlcV9ubyI6MCwiX3ByaW1hcnlfdGVybSI6MSwic3RhdHVzIjoyMDF9fSx7ImluZGV4Ijp7Il9pbmRleCI6Imdlb25hbWVzIiwiX2lkIjoiMTRfNkM1SUIzdGszODYtVG5NOUYiLCJfdmVyc2lvbiI6MSwicmVzdWx0IjoiY3JlYXRlZCIsIl9zaGFyZHMiOnsidG90YWwiOjEsInN1Y2Nlc3NmdWwiOjEsImZhaWxlZCI6MH0sIl9zZXFfbm8iOjEsIl9wcmltYXJ5X3Rlcm0iOjEsInN0YXR1cyI6MjAxfX0seyJpbmRleCI6eyJfaW5kZXgiOiJnZW9uYW1lcyIsIl9pZCI6IjJJXzZDNUlCM3RrMzg2LVRuTTlGIiwiX3ZlcnNpb24iOjEsInJlc3VsdCI6ImNyZWF0ZWQiLCJfc2hhcmRzIjp7InRvdGFsIjoxLCJzdWNjZXNzZnVsIjoxLCJmYWlsZWQiOjB9LCJfc2VxX25vIjoyLCJfcHJpbWFyeV90ZXJtIjoxLCJzdGF0dXMiOjIwMX19LHsiaW5kZXgiOnsiX2luZGV4IjoiZ2VvbmFtZXMiLCJfaWQiOiIyWV82QzVJQjN0azM4Ni1Ubk05RiIsIl92ZXJzaW9uIjoxLCJyZXN1bHQiOiJjcmVhdGVkIiwiX3NoYXJkcyI6eyJ0b3RhbCI6MSwic3VjY2Vzc2Z1bCI6MSwiZmFpbGVkIjowfSwiX3NlcV9ubyI6MywiX3ByaW1hcnlfdGVybSI6MSwic3RhdHVzIjoyMDF9fSx7ImluZGV4Ijp7Il9pbmRleCI6Imdlb25hbWVzIiwiX2lkIjoiMm9fNkM1SUIzdGszODYtVG5NOUYiLCJfdmVyc2lvbiI6MSwicmVzdWx0IjoiY3JlYXRlZCIsIl9zaGFyZHMiOnsidG90YWwiOjEsInN1Y2Nlc3NmdWwiOjEsImZhaWxlZCI6MH0sIl9zZXFfbm8iOjAsIl9wcmltYXJ5X3Rlcm0iOjEsInN0YXR1cyI6MjAxfX1dfQ==" + } + ], + "connectionId": "0242acfffe12000b-0000000a-0000000d-380cb36fdefa9c00-e2a1d15a.0", + "numRequests": 1, + "numErrors": 0 + } +] diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/multiple_tuples_parsed.json b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/multiple_tuples_parsed.json new file mode 100644 index 000000000..464afd0cb --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/multiple_tuples_parsed.json @@ -0,0 +1,1018 @@ +[ + { + "sourceRequest": { + "Request-URI": "/", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "User-Agent": "python-requests/2.32.3", + "Accept-Encoding": "gzip, deflate, zstd", + "Accept": "*/*", + "Connection": "keep-alive", + "Authorization": "Basic YWRtaW46YWRtaW4=", + "body": "" + }, + "sourceResponse": { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 374, + "content-type": "application/json; charset=UTF-8", + "content-length": "538", + "body": { + "name": "3cc068ad54eb", + "cluster_name": "docker-cluster", + "cluster_uuid": "PIAawDwSQKSRLtoaXC-oWg", + "version": { + "number": "7.10.2", + "build_flavor": "oss", + "build_type": "docker", + "build_hash": "747e1cc71def077253878a59143c1f785afa92b9", + "build_date": "2021-01-13T00:42:12.435326Z", + "build_snapshot": false, + "lucene_version": "8.7.0", + "minimum_wire_compatibility_version": "6.8.0", + "minimum_index_compatibility_version": "6.0.0-beta1" + }, + "tagline": "You Know, for Search" + } + }, + "targetRequest": { + "Request-URI": "/", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "User-Agent": "python-requests/2.32.3", + "Accept-Encoding": "gzip, deflate, zstd", + "Accept": "*/*", + "Connection": "keep-alive", + "Authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "body": "" + }, + "targetResponses": [ + { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 341, + "content-type": "application/json; charset=UTF-8", + "content-length": "568", + "body": { + "name": "e6b3a4370763", + "cluster_name": "docker-cluster", + "cluster_uuid": "88zlik7ORFuFT78zduradA", + "version": { + "distribution": "opensearch", + "number": "2.15.0", + "build_type": "tar", + "build_hash": "61dbcd0795c9bfe9b81e5762175414bc38bbcadf", + "build_date": "2024-06-20T03:27:32.562036890Z", + "build_snapshot": false, + "lucene_version": "9.10.0", + "minimum_wire_compatibility_version": "7.10.0", + "minimum_index_compatibility_version": "7.0.0" + }, + "tagline": "The OpenSearch Project: https://opensearch.org/" + } + } + ], + "connectionId": "0242acfffe12000b-0000000a-00000003-4c7bb8f4aefa3189-c0544dbf.0", + "numRequests": 1, + "numErrors": 0 + }, + { + "sourceRequest": { + "Request-URI": "/geonames", + "Method": "PUT", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46YWRtaW4=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "1214", + "body": { + "settings": { + "index.number_of_shards": 5, + "index.number_of_replicas": 0, + "index.store.type": "fs", + "index.queries.cache.enabled": false, + "index.requests.cache.enable": false + }, + "mappings": { + "dynamic": "strict", + "_source": { "enabled": true }, + "properties": { + "elevation": { "type": "integer" }, + "name": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "geonameid": { "type": "long" }, + "feature_class": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "location": { "type": "geo_point" }, + "cc2": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "timezone": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "dem": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "country_code": { + "type": "text", + "fielddata": true, + "fields": { "raw": { "type": "keyword" } } + }, + "admin1_code": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "admin2_code": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "admin3_code": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "admin4_code": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "feature_code": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "alternatenames": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "asciiname": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "population": { "type": "long" } + } + } + } + }, + "sourceResponse": { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 232, + "content-type": "application/json; charset=UTF-8", + "content-length": "67", + "body": { + "acknowledged": true, + "shards_acknowledged": true, + "index": "geonames" + } + }, + "targetRequest": { + "Request-URI": "/geonames", + "Method": "PUT", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "1214", + "body": { + "settings": { + "index.number_of_shards": 5, + "index.number_of_replicas": 0, + "index.store.type": "fs", + "index.queries.cache.enabled": false, + "index.requests.cache.enable": false + }, + "mappings": { + "dynamic": "strict", + "_source": { "enabled": true }, + "properties": { + "elevation": { "type": "integer" }, + "name": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "geonameid": { "type": "long" }, + "feature_class": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "location": { "type": "geo_point" }, + "cc2": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "timezone": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "dem": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "country_code": { + "type": "text", + "fielddata": true, + "fields": { "raw": { "type": "keyword" } } + }, + "admin1_code": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "admin2_code": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "admin3_code": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "admin4_code": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "feature_code": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "alternatenames": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "asciiname": { + "type": "text", + "fields": { "raw": { "type": "keyword" } } + }, + "population": { "type": "long" } + } + } + } + }, + "targetResponses": [ + { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 270, + "content-type": "application/json; charset=UTF-8", + "content-length": "67", + "body": { + "acknowledged": true, + "shards_acknowledged": true, + "index": "geonames" + } + } + ], + "connectionId": "0242acfffe12000b-0000000a-00000009-19ac20d3defa9804-07697cc6.0", + "numRequests": 1, + "numErrors": 0 + }, + { + "sourceRequest": { + "Request-URI": "/_cluster/health/geonames?wait_for_status=green&wait_for_no_relocating_shards=true", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46YWRtaW4=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "body": "" + }, + "sourceResponse": { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 3, + "content-type": "application/json; charset=UTF-8", + "content-length": "390", + "body": { + "cluster_name": "docker-cluster", + "status": "green", + "timed_out": false, + "number_of_nodes": 1, + "number_of_data_nodes": 1, + "active_primary_shards": 5, + "active_shards": 5, + "relocating_shards": 0, + "initializing_shards": 0, + "unassigned_shards": 0, + "delayed_unassigned_shards": 0, + "number_of_pending_tasks": 0, + "number_of_in_flight_fetch": 0, + "task_max_waiting_in_queue_millis": 0, + "active_shards_percent_as_number": 100.0 + } + }, + "targetRequest": { + "Request-URI": "/_cluster/health/geonames?wait_for_status=green&wait_for_no_relocating_shards=true", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "body": "" + }, + "targetResponses": [ + { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 29, + "content-type": "application/json; charset=UTF-8", + "content-length": "449", + "body": { + "cluster_name": "docker-cluster", + "status": "green", + "timed_out": false, + "number_of_nodes": 1, + "number_of_data_nodes": 1, + "discovered_master": true, + "discovered_cluster_manager": true, + "active_primary_shards": 5, + "active_shards": 5, + "relocating_shards": 0, + "initializing_shards": 0, + "unassigned_shards": 0, + "delayed_unassigned_shards": 0, + "number_of_pending_tasks": 0, + "number_of_in_flight_fetch": 0, + "task_max_waiting_in_queue_millis": 0, + "active_shards_percent_as_number": 100.0 + } + } + ], + "connectionId": "0242acfffe12000b-0000000a-0000000b-2681c86bdefa99ec-45c6b416.0", + "numRequests": 1, + "numErrors": 0 + }, + { + "sourceRequest": { + "Request-URI": "/_bulk", + "Method": "POST", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46YWRtaW4=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "3974", + "body": [ + { "index": { "_index": "geonames" } }, + { + "geonameid": 2986043, + "name": "Pic de Font Blanca", + "asciiname": "Pic de Font Blanca", + "alternatenames": "Pic de Font Blanca,Pic du Port", + "feature_class": "T", + "feature_code": "PK", + "country_code": "AD", + "admin1_code": "00", + "population": 0, + "dem": "2860", + "timezone": "Europe/Andorra", + "location": [1.53335, 42.64991] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 2993838, + "name": "Pic de Mil-Menut", + "asciiname": "Pic de Mil-Menut", + "alternatenames": "Pic de Mil-Menut", + "feature_class": "T", + "feature_code": "PK", + "country_code": "AD", + "cc2": "AD,FR", + "admin1_code": "B3", + "admin2_code": "09", + "admin3_code": "091", + "admin4_code": "09024", + "population": 0, + "dem": "2138", + "timezone": "Europe/Andorra", + "location": [1.65, 42.63333] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 2994701, + "name": "Roc M\u00e9l\u00e9", + "asciiname": "Roc Mele", + "alternatenames": "Roc Mele,Roc Meler,Roc M\u00e9l\u00e9", + "feature_class": "T", + "feature_code": "MT", + "country_code": "AD", + "cc2": "AD,FR", + "admin1_code": "00", + "population": 0, + "dem": "2803", + "timezone": "Europe/Andorra", + "location": [1.74028, 42.58765] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3007683, + "name": "Pic des Langounelles", + "asciiname": "Pic des Langounelles", + "alternatenames": "Pic des Langounelles", + "feature_class": "T", + "feature_code": "PK", + "country_code": "AD", + "cc2": "AD,FR", + "admin1_code": "00", + "population": 0, + "dem": "2685", + "timezone": "Europe/Andorra", + "location": [1.47364, 42.61203] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3017832, + "name": "Pic de les Abelletes", + "asciiname": "Pic de les Abelletes", + "alternatenames": "Pic de la Font-Negre,Pic de la Font-N\u00e8gre,Pic de les Abelletes", + "feature_class": "T", + "feature_code": "PK", + "country_code": "AD", + "cc2": "FR", + "admin1_code": "A9", + "admin2_code": "66", + "admin3_code": "663", + "admin4_code": "66146", + "population": 0, + "dem": "2411", + "timezone": "Europe/Andorra", + "location": [1.73343, 42.52535] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3017833, + "name": "Estany de les Abelletes", + "asciiname": "Estany de les Abelletes", + "alternatenames": "Estany de les Abelletes,Etang de Font-Negre,\u00c9tang de Font-N\u00e8gre", + "feature_class": "H", + "feature_code": "LK", + "country_code": "AD", + "cc2": "FR", + "admin1_code": "A9", + "population": 0, + "dem": "2260", + "timezone": "Europe/Andorra", + "location": [1.73362, 42.52915] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3023203, + "name": "Port Vieux de la Coume d\u2019Ose", + "asciiname": "Port Vieux de la Coume d'Ose", + "alternatenames": "Port Vieux de Coume d'Ose,Port Vieux de Coume d\u2019Ose,Port Vieux de la Coume d'Ose,Port Vieux de la Coume d\u2019Ose", + "feature_class": "T", + "feature_code": "PASS", + "country_code": "AD", + "admin1_code": "00", + "population": 0, + "dem": "2687", + "timezone": "Europe/Andorra", + "location": [1.61823, 42.62568] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3029315, + "name": "Port de la Cabanette", + "asciiname": "Port de la Cabanette", + "alternatenames": "Port de la Cabanette,Porteille de la Cabanette", + "feature_class": "T", + "feature_code": "PASS", + "country_code": "AD", + "cc2": "AD,FR", + "admin1_code": "B3", + "admin2_code": "09", + "admin3_code": "091", + "admin4_code": "09139", + "population": 0, + "dem": "2379", + "timezone": "Europe/Andorra", + "location": [1.73333, 42.6] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3034945, + "name": "Port Dret", + "asciiname": "Port Dret", + "alternatenames": "Port Dret,Port de Bareites,Port de las Bareytes,Port des Bareytes", + "feature_class": "T", + "feature_code": "PASS", + "country_code": "AD", + "admin1_code": "00", + "population": 0, + "dem": "2660", + "timezone": "Europe/Andorra", + "location": [1.45562, 42.60172] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3038814, + "name": "Costa de Xurius", + "asciiname": "Costa de Xurius", + "feature_class": "T", + "feature_code": "SLP", + "country_code": "AD", + "admin1_code": "07", + "population": 0, + "dem": "1839", + "timezone": "Europe/Andorra", + "location": [1.47569, 42.50692] + } + ] + }, + "sourceResponse": { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 47, + "content-type": "application/json; charset=UTF-8", + "content-length": "2026", + "body": [ + { + "took": 35, + "errors": false, + "items": [ + { + "index": { + "_index": "geonames", + "_type": "_doc", + "_id": "I9D6C5IBHCDt93k-nBIw", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 0, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_type": "_doc", + "_id": "JND6C5IBHCDt93k-nBIx", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 0, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_type": "_doc", + "_id": "JdD6C5IBHCDt93k-nBIx", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 1, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_type": "_doc", + "_id": "JtD6C5IBHCDt93k-nBIx", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 0, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_type": "_doc", + "_id": "J9D6C5IBHCDt93k-nBIx", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 0, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_type": "_doc", + "_id": "KND6C5IBHCDt93k-nBIx", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 2, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_type": "_doc", + "_id": "KdD6C5IBHCDt93k-nBIx", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 1, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_type": "_doc", + "_id": "KtD6C5IBHCDt93k-nBIx", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 3, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_type": "_doc", + "_id": "K9D6C5IBHCDt93k-nBIx", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 1, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_type": "_doc", + "_id": "LND6C5IBHCDt93k-nBIx", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 0, + "_primary_term": 1, + "status": 201 + } + } + ] + } + ] + }, + "targetRequest": { + "Request-URI": "/_bulk", + "Method": "POST", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "content-type": "application/json", + "user-agent": "opensearch-py/2.6.0 (Python 3.11.6)", + "authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "Connection": "keep-alive", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "3974", + "body": [ + { "index": { "_index": "geonames" } }, + { + "geonameid": 2986043, + "name": "Pic de Font Blanca", + "asciiname": "Pic de Font Blanca", + "alternatenames": "Pic de Font Blanca,Pic du Port", + "feature_class": "T", + "feature_code": "PK", + "country_code": "AD", + "admin1_code": "00", + "population": 0, + "dem": "2860", + "timezone": "Europe/Andorra", + "location": [1.53335, 42.64991] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 2993838, + "name": "Pic de Mil-Menut", + "asciiname": "Pic de Mil-Menut", + "alternatenames": "Pic de Mil-Menut", + "feature_class": "T", + "feature_code": "PK", + "country_code": "AD", + "cc2": "AD,FR", + "admin1_code": "B3", + "admin2_code": "09", + "admin3_code": "091", + "admin4_code": "09024", + "population": 0, + "dem": "2138", + "timezone": "Europe/Andorra", + "location": [1.65, 42.63333] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 2994701, + "name": "Roc M\u00e9l\u00e9", + "asciiname": "Roc Mele", + "alternatenames": "Roc Mele,Roc Meler,Roc M\u00e9l\u00e9", + "feature_class": "T", + "feature_code": "MT", + "country_code": "AD", + "cc2": "AD,FR", + "admin1_code": "00", + "population": 0, + "dem": "2803", + "timezone": "Europe/Andorra", + "location": [1.74028, 42.58765] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3007683, + "name": "Pic des Langounelles", + "asciiname": "Pic des Langounelles", + "alternatenames": "Pic des Langounelles", + "feature_class": "T", + "feature_code": "PK", + "country_code": "AD", + "cc2": "AD,FR", + "admin1_code": "00", + "population": 0, + "dem": "2685", + "timezone": "Europe/Andorra", + "location": [1.47364, 42.61203] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3017832, + "name": "Pic de les Abelletes", + "asciiname": "Pic de les Abelletes", + "alternatenames": "Pic de la Font-Negre,Pic de la Font-N\u00e8gre,Pic de les Abelletes", + "feature_class": "T", + "feature_code": "PK", + "country_code": "AD", + "cc2": "FR", + "admin1_code": "A9", + "admin2_code": "66", + "admin3_code": "663", + "admin4_code": "66146", + "population": 0, + "dem": "2411", + "timezone": "Europe/Andorra", + "location": [1.73343, 42.52535] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3017833, + "name": "Estany de les Abelletes", + "asciiname": "Estany de les Abelletes", + "alternatenames": "Estany de les Abelletes,Etang de Font-Negre,\u00c9tang de Font-N\u00e8gre", + "feature_class": "H", + "feature_code": "LK", + "country_code": "AD", + "cc2": "FR", + "admin1_code": "A9", + "population": 0, + "dem": "2260", + "timezone": "Europe/Andorra", + "location": [1.73362, 42.52915] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3023203, + "name": "Port Vieux de la Coume d\u2019Ose", + "asciiname": "Port Vieux de la Coume d'Ose", + "alternatenames": "Port Vieux de Coume d'Ose,Port Vieux de Coume d\u2019Ose,Port Vieux de la Coume d'Ose,Port Vieux de la Coume d\u2019Ose", + "feature_class": "T", + "feature_code": "PASS", + "country_code": "AD", + "admin1_code": "00", + "population": 0, + "dem": "2687", + "timezone": "Europe/Andorra", + "location": [1.61823, 42.62568] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3029315, + "name": "Port de la Cabanette", + "asciiname": "Port de la Cabanette", + "alternatenames": "Port de la Cabanette,Porteille de la Cabanette", + "feature_class": "T", + "feature_code": "PASS", + "country_code": "AD", + "cc2": "AD,FR", + "admin1_code": "B3", + "admin2_code": "09", + "admin3_code": "091", + "admin4_code": "09139", + "population": 0, + "dem": "2379", + "timezone": "Europe/Andorra", + "location": [1.73333, 42.6] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3034945, + "name": "Port Dret", + "asciiname": "Port Dret", + "alternatenames": "Port Dret,Port de Bareites,Port de las Bareytes,Port des Bareytes", + "feature_class": "T", + "feature_code": "PASS", + "country_code": "AD", + "admin1_code": "00", + "population": 0, + "dem": "2660", + "timezone": "Europe/Andorra", + "location": [1.45562, 42.60172] + }, + { "index": { "_index": "geonames" } }, + { + "geonameid": 3038814, + "name": "Costa de Xurius", + "asciiname": "Costa de Xurius", + "feature_class": "T", + "feature_code": "SLP", + "country_code": "AD", + "admin1_code": "07", + "population": 0, + "dem": "1839", + "timezone": "Europe/Andorra", + "location": [1.47569, 42.50692] + } + ] + }, + "targetResponses": [ + { + "HTTP-Version": { "keepAliveDefault": true }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 49, + "content-type": "application/json; charset=UTF-8", + "content-length": "1876", + "body": [ + { + "took": 25, + "errors": false, + "items": [ + { + "index": { + "_index": "geonames", + "_id": "0Y_6C5IB3tk386-TnM9F", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 0, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_id": "0o_6C5IB3tk386-TnM9F", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 0, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_id": "04_6C5IB3tk386-TnM9F", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 1, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_id": "1I_6C5IB3tk386-TnM9F", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 2, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_id": "1Y_6C5IB3tk386-TnM9F", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 0, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_id": "1o_6C5IB3tk386-TnM9F", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 0, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_id": "14_6C5IB3tk386-TnM9F", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 1, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_id": "2I_6C5IB3tk386-TnM9F", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 2, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_id": "2Y_6C5IB3tk386-TnM9F", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 3, + "_primary_term": 1, + "status": 201 + } + }, + { + "index": { + "_index": "geonames", + "_id": "2o_6C5IB3tk386-TnM9F", + "_version": 1, + "result": "created", + "_shards": { "total": 1, "successful": 1, "failed": 0 }, + "_seq_no": 0, + "_primary_term": 1, + "status": 201 + } + } + ] + } + ] + } + ], + "connectionId": "0242acfffe12000b-0000000a-0000000d-380cb36fdefa9c00-e2a1d15a.0", + "numRequests": 1, + "numErrors": 0 + } +] diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/valid_tuple.json b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/valid_tuple.json new file mode 100644 index 000000000..8d7af4d13 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/valid_tuple.json @@ -0,0 +1,49 @@ +{ + "sourceRequest": { + "Request-URI": "/_cat/indices?v", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "Authorization": "Basic YWRtaW46YWRtaW4=", + "User-Agent": "curl/8.5.0", + "Accept": "*/*", + "body": "" + }, + "sourceResponse": { + "HTTP-Version": { + "keepAliveDefault": true + }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 59, + "content-type": "text/plain; charset=UTF-8", + "content-length": "214", + "body": "aGVhbHRoIHN0YXR1cyBpbmRleCAgICAgICB1dWlkICAgICAgICAgICAgICAgICAgIHByaSByZXAgZG9jcy5jb3VudCBkb2NzLmRlbGV0ZWQgc3RvcmUuc2l6ZSBwcmkuc3RvcmUuc2l6ZQpncmVlbiAgb3BlbiAgIHNlYXJjaGd1YXJkIHlKS1hQTUh0VFJPTklYU1pYQ193bVEgICAxICAgMCAgICAgICAgICA4ICAgICAgICAgICAgMCAgICAgNDQuN2tiICAgICAgICAgNDQuN2tiCg==" + }, + "targetRequest": { + "Request-URI": "/_cat/indices?v", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "Authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "User-Agent": "curl/8.5.0", + "Accept": "*/*", + "body": "" + }, + "targetResponses": [ + { + "HTTP-Version": { + "keepAliveDefault": true + }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 721, + "content-type": "text/plain; charset=UTF-8", + "content-length": "484", + "body": "aGVhbHRoIHN0YXR1cyBpbmRleCAgICAgICAgICAgICAgICAgICAgIHV1aWQgICAgICAgICAgICAgICAgICAgcHJpIHJlcCBkb2NzLmNvdW50IGRvY3MuZGVsZXRlZCBzdG9yZS5zaXplIHByaS5zdG9yZS5zaXplCmdyZWVuICBvcGVuICAgLm9wZW5zZWFyY2gtb2JzZXJ2YWJpbGl0eSA4Vy1vWUhmYlN5U3JkeFFFX3NPbnpnICAgMSAgIDAgICAgICAgICAgMCAgICAgICAgICAgIDAgICAgICAgMjA4YiAgICAgICAgICAgMjA4YgpncmVlbiAgb3BlbiAgIC5wbHVnaW5zLW1sLWNvbmZpZyAgICAgICAgRjludnh2c2dSelNibG1mSnZ2aGptdyAgIDEgICAwICAgICAgICAgIDEgICAgICAgICAgICAwICAgICAgMy44a2IgICAgICAgICAgMy44a2IKZ3JlZW4gIG9wZW4gICAub3BlbmRpc3Ryb19zZWN1cml0eSAgICAgIDVmWHlhbkZuU2tDUUQ2bjFKUW1KTlEgICAxICAgMCAgICAgICAgIDEwICAgICAgICAgICAgMCAgICAgNzcuNWtiICAgICAgICAgNzcuNWtiCg==" + } + ], + "connectionId": "0242acfffe13000a-0000000a-00000005-1eb087a9beb83f3e-a32794b4.0", + "numRequests": 1, + "numErrors": 0 +} diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/valid_tuple_parsed.json b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/valid_tuple_parsed.json new file mode 100644 index 000000000..c587208ab --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/data/valid_tuple_parsed.json @@ -0,0 +1,49 @@ +{ + "sourceRequest": { + "Request-URI": "/_cat/indices?v", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "capture-proxy:9200", + "Authorization": "Basic YWRtaW46YWRtaW4=", + "User-Agent": "curl/8.5.0", + "Accept": "*/*", + "body": "" + }, + "sourceResponse": { + "HTTP-Version": { + "keepAliveDefault": true + }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 59, + "content-type": "text/plain; charset=UTF-8", + "content-length": "214", + "body": "health status index uuid pri rep docs.count docs.deleted store.size pri.store.size\ngreen open searchguard yJKXPMHtTRONIXSZXC_wmQ 1 0 8 0 44.7kb 44.7kb\n" + }, + "targetRequest": { + "Request-URI": "/_cat/indices?v", + "Method": "GET", + "HTTP-Version": "HTTP/1.1", + "Host": "opensearchtarget", + "Authorization": "Basic YWRtaW46bXlTdHJvbmdQYXNzd29yZDEyMyE=", + "User-Agent": "curl/8.5.0", + "Accept": "*/*", + "body": "" + }, + "targetResponses": [ + { + "HTTP-Version": { + "keepAliveDefault": true + }, + "Status-Code": 200, + "Reason-Phrase": "OK", + "response_time_ms": 721, + "content-type": "text/plain; charset=UTF-8", + "content-length": "484", + "body": "health status index uuid pri rep docs.count docs.deleted store.size pri.store.size\ngreen open .opensearch-observability 8W-oYHfbSySrdxQE_sOnzg 1 0 0 0 208b 208b\ngreen open .plugins-ml-config F9nvxvsgRzSblmfJvvhjmw 1 0 1 0 3.8kb 3.8kb\ngreen open .opendistro_security 5fXyanFnSkCQD6n1JQmJNQ 1 0 10 0 77.5kb 77.5kb\n" + } + ], + "connectionId": "0242acfffe13000a-0000000a-00000005-1eb087a9beb83f3e-a32794b4.0", + "numRequests": 1, + "numErrors": 0 +} \ No newline at end of file diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_cli.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_cli.py index 6ace00371..51305fb5a 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_cli.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_cli.py @@ -608,3 +608,33 @@ def test_cli_kafka_describe_topic(runner, mocker): def test_completion_script(runner): result = runner.invoke(cli, [str(VALID_SERVICES_YAML), 'completion', 'bash'], catch_exceptions=True) assert result.exit_code == 0 + + +def test_tuple_converter(runner, tmp_path): + # The `multiple_tuples` and `multiple_tuples_parsed` files are formatted as "real" json objects so that + # they can be pretty-printed and examined, but they need to be converted to ndjson files to be used by the + # CLI command. + + # Make the input file + input_tuples_file = f"{TEST_DATA_DIRECTORY}/multiple_tuples.json" + with open(input_tuples_file, 'r') as f: + input_tuples = json.load(f) + ndjson_input_file = f"{tmp_path}/tuples.ndjson" + with open(ndjson_input_file, 'w') as f: + f.write('\n'.join([json.dumps(record) for record in input_tuples])) + + ndjson_output_file = f"{tmp_path}/converted_tuples.ndjson" + result = runner.invoke(cli, ['--config-file', str(VALID_SERVICES_YAML), 'tuples', 'show', + '--in', ndjson_input_file, + '--out', ndjson_output_file], + catch_exceptions=True) + assert ndjson_output_file in result.output + assert result.exit_code == 0 + + # Open the ndjson output file and compare it to the "real" output file + expected_output_file = f"{TEST_DATA_DIRECTORY}/multiple_tuples_parsed.json" + with open(expected_output_file, 'r') as f: + output_tuples = json.load(f) + expected_output_as_ndjson = [json.dumps(record) + "\n" for record in output_tuples] + + assert open(ndjson_output_file).readlines() == expected_output_as_ndjson diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_tuple_reader.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_tuple_reader.py new file mode 100644 index 000000000..7f24cc49e --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_tuple_reader.py @@ -0,0 +1,163 @@ +import json +import pathlib +import re + +import pytest + +from console_link.models.tuple_reader import (DictionaryPathException, get_element_with_regex, get_element, + set_element, Flag, get_flags_for_component, + parse_tuple) + + +TEST_DATA_DIRECTORY = pathlib.Path(__file__).parent / "data" +VALID_TUPLE = TEST_DATA_DIRECTORY / "valid_tuple.json" +VALID_TUPLE_PARSED = TEST_DATA_DIRECTORY / "valid_tuple_parsed.json" +VALID_TUPLE_GZIPPED_CHUNKED = TEST_DATA_DIRECTORY / "valid_tuple_gzipped_and_chunked.json" +VALID_TUPLE_GZIPPED_CHUNKED_PARSED = TEST_DATA_DIRECTORY / "valid_tuple_gzipped_and_chunked_parsed.json" +INVALID_TUPLE = TEST_DATA_DIRECTORY / "invalid_tuple.json" + + +def test_get_element_with_regex_succeeds(): + d = { + 'A1': 'value', + '2B': 'not value' + } + regex = re.compile(r"\w\d") + assert get_element_with_regex(regex, d) == "value" + + +def test_get_element_with_regex_fails_no_raise(): + d = { + 'AA1': 'value', + '2B': 'not value' + } + regex = re.compile(r"\w\d") + assert get_element_with_regex(regex, d) is None + + +def test_get_element_with_regex_fails_with_raise(): + d = { + 'AA1': 'value', + '2B': 'not value' + } + regex = re.compile(r"\w\d") + with pytest.raises(DictionaryPathException): + get_element_with_regex(regex, d, raise_on_error=True) + + +def test_get_element_succeeds(): + d = { + 'A': { + 'B': 'value', + 'C': 'not value' + }, + 'B': 'not value' + } + assert get_element('A.B', d) == 'value' + + +def test_get_element_fails_no_raise(): + d = { + 'A': { + 'B': 'value', + 'C': 'not value' + }, + 'B': 'not value' + } + assert get_element('B.A', d) is None + + +def test_get_element_fails_raises(): + d = { + 'A': { + 'B': 'value', + 'C': 'not value' + }, + 'B': 'not value' + } + with pytest.raises(DictionaryPathException): + get_element('B.A', d, raise_on_error=True) + + +def test_set_element_succeeds(): + d = { + 'A': { + 'B': 'value', + 'C': 'not value' + }, + 'B': 'not value' + } + set_element('A.B', d, 'new value') + + assert d['A']['B'] == 'new value' + + +def test_set_element_fails_and_raises(): + d = { + 'A': { + 'B': 'value', + 'C': 'not value' + }, + 'B': 'not value' + } + with pytest.raises(DictionaryPathException): + set_element('B.A', d, 'new value') + + +def test_get_flags_none(): + request = { + 'Content-Encoding': 'not-gzip', + 'Content-Type': 'not-json', + 'Transfer-Encoding': 'not-chunked', + 'body': 'abcdefg' + } + flags = get_flags_for_component(request, False) + assert flags == set() + + +def test_get_flags_for_component_only_bulk(): + request = { + 'Content-Type': 'not-json', + 'body': 'abcdefg' + } + flags = get_flags_for_component(request, True) + assert flags == {Flag.Bulk_Request} + + +def test_get_flags_for_component_all_present(): + request = { + 'Content-Type': 'application/json', + 'body': 'abcdefg' + } + flags = get_flags_for_component(request, True) + assert flags == {Flag.Bulk_Request, Flag.Json} + + +def test_get_flags_all_present_alternate_capitalization(): + request = { + 'CONTENT-TYPE': 'application/json', + 'body': 'abcdefg' + } + flags = get_flags_for_component(request, True) + assert flags == {Flag.Bulk_Request, Flag.Json} + + +def test_parse_tuple_full_example(): + with open(VALID_TUPLE, 'r') as f: + tuple_ = f.read() + parsed = parse_tuple(tuple_, 0) + + with open(VALID_TUPLE_PARSED, 'r') as f: + expected = json.load(f) + + assert parsed == expected + + +def test_parse_tuple_with_malformed_bodies(caplog): + with open(INVALID_TUPLE, 'r') as f: + tuple_ = f.read() + + parsed = parse_tuple(tuple_, 0) + assert json.loads(tuple_) == parsed # Values weren't changed if they couldn't be interpreted + assert "Body value of sourceResponse on line 0 could not be decoded to utf-8" in caplog.text + assert "Body value of targetResponses item 0 on line 0 should be a json, but could not be parsed" in caplog.text