diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000000000..feed8effe759ac --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +CMakeCache.txt \ No newline at end of file diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 00000000000000..0d20b6487c61e7 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/tests/Dockerfile b/tests/Dockerfile new file mode 100644 index 00000000000000..7081846c7b918e --- /dev/null +++ b/tests/Dockerfile @@ -0,0 +1,16 @@ +FROM alpine as build-env +RUN apk add build-base && apk add cmake +WORKDIR /app + +COPY . . +COPY ./tests/ci-test.sh . +# Compile the binaries +RUN cmake . && make + +FROM alpine +RUN apk add iproute2-tc +COPY --from=build-env /app/udpst /app/udpst +COPY --from=build-env /app/ci-test.sh /app/ci-test.sh +RUN chmod +x /app/ci-test.sh +WORKDIR /app +ENTRYPOINT ["/app/ci-test.sh"] diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000000000..e2e82c86b3fcac --- /dev/null +++ b/tests/README.md @@ -0,0 +1,360 @@ +# OB-UDPST Functional Testing + +The following framework is provided for running automated functional tests on OB-UDPST. +The basic premise of the automated testing is through the use of containers in +combination with NetEm. The server container runs udpst as the server process, after a +NetEm configuration is applied to the container virtual Ethernet interface, thus setting +the downstream network emulation parameters. This process is repeated on the client +container for upstream. The udpst client then runs the speed-test over the connection +between the two containers, measuring the performance of the network, including any +emulated WAN settings (i.e. rate, loss, jitter, etc.). + +This framework allows developers to define test cases through the `test_cases.yaml` +file. Each test case definition provides three key components: + +* A set of UDPST command line flags for the service and client (Note 1) +* A set of NetEm configuration values for upstream and downstream +* A set of metrics to evaluate the measurements from UDPST + +## Requirements + +The server and client commands **MUST** include the appropriate flags to run as the +expected mode. For example, the `client-cli` command flags **MUST** include `"-d"` or +`"-d server"` to run in the client mode. Additionally the `client-cli` value **MUST** +include `"-f -json"` if metrics are defined for a specific test case within `test_cases.yaml` + +The system running the test must provide a set of requirements, including: + +* docker +* docker compose v2 +* python +* python modules from requirements.txt + +An example bash script `setup.sh` to create the python virtual environment and install +the requirements. + +## Defining the Tests + +**NOTE:** At this time there is no schema validation enforced for the `test_cases.yaml` +file prior to parsing the `key: value` pairs. + +The information model for a test case: +```yml +--- +- case-1: + client-cli: "client cli command options" + server-cli: "server cli command options" + netem: # full set of netem options, will not be a valid netem command + downstream: + limit: 1 # in packets + delay: + params: + - 2 # in ms + - 3 # in ms, jitter (optional) + - 4 # in percent, correlation (optional) + distribution: normal # { uniform | normal | pareto | paretonormal } (optional) + reorder: + params: + - 5 # in percent (optional) + - 6 # in percent, correlation (optional) + gap: 7 # as an integer (optional) + loss: + params: ecn # flag to use Explicit Congestion Notification + random: 8 # in percent + state: + - 9 # in percent, p13 + - 10 # in percent, p31 (optional) + - 11 # in percent, p32 (optional) + - 12 # in percent, p23 (optional) + - 13 # in percent, p14 (optional) + gemodel: + - 14 # in percent, p + - 15 # in percent, r (optional) + - 16 # in percent, 1-h (optional) + - 17 # in percent, 1-k (optional) + corrupt: + - 18 # in percent + - 19 # in percent, correlation (optional) + duplicate: + - 20 # in percent + - 21 # in percent, correlation (optional) + rate: + - 22 # in bits per second + - 23 # in bytes (can be negative), packetoverhead (optional) + - 24 # as an unsigned integer, cellsize (optional) + - 25 # as an integer (can be negative), celloverhead (optional) + slot: + params: + - 26 # assume default is in us, min_delay + - 27 # assume default is in us, max_delay, (optional) + distribution: + - normal # { uniform | normal | pareto | paretonormal | custom} (optional) + - 28 # in ms, delay + - 29 # in ms, jitter + packets: 30 # in packets, (optional) + bytes: 31 # in bytes, (optional) + upstream: + limit: 10 # in packets + delay: + params: + - 100 # in ms + - 10 # in ms, jitter (optional) + - 1 # in percent, correlation (optional) + distribution: normal # { uniform | normal | pareto | paretonormal } (optional) + reorder: + params: + - 25 # in percent + - 75 # in percent, correlation (optional) + gap: 4 # as an integer (optional) + loss: + params: ecn # flag to use Explicit Congestion Notification + random: 0.1 # in percent + state: + - 1 # in percent, p13 + - 5 # in percent, p31 (optional) + - 10 # in percent, p32 (optional) + - 50 # in percent, p23 (optional) + - 2 # in percent, p14 (optional) + gemodel: + - 1 # in percent, p + - 5 # in percent, r (optional) + - 10 # in percent, 1-h (optional) + - 10 # in percent, 1-k (optional) + corrupt: + - 1 # in percent + - 3 # in percent, correlation (optional) + duplicate: + - 1 # in percent + - 3 # in percent, correlation (optional) + rate: + - 1 # in bits per second + - 2 # in bytes (can be negative), packetoverhead (optional) + - 3 # as an unsigned integer, cellsize (optional) + - 5 # as an integer (can be negative), celloverhead (optional) + slot: + params: + - 800 # assume default is in us, min_delay + - 1000 # assume default is in us, max_delay, (optional) + distribution: + - normal # { uniform | normal | pareto | paretonormal | custom} (optional) + - 100 # in ms, delay + - 10 # in ms, jitter + packets: 10 # in packets, (optional) + bytes: 3 # in bytes, (optional) + metrics: + loss_lessthan_10: results["Output"]["Summary"]["DeliveredPercent"] > 90.0 + rtt_range_consistent: results["Output"]["Summary"]["RTTMax"] - results["Output"]["Summary"]["RTTMin"] < 1.0 +... +``` + +### Test Cases + +All test case keys should be unique and provide some context as to what the test case is +testing for readability after all tests are completed. + +### CLI Options + +Strings defining the client and server CLI commands, per UDPST. + +### NetEm Options + +For a detailed description of NetEm options see [tc-netem(8) — Linux manual page](https://man7.org/linux/man-pages/man8/tc-netem.8.html "NetEm man page") and [networking:netem [Wiki]](https://wiki.linuxfoundation.org/networking/netem "Linux Foundation NetEm Wiki"). + +It should be noted that while all portions of the `netem` key can be filled in, there are +some combinations of `key: value` pairs that will result in invalid NetEm command +strings. i.e. specifying a `reorder` value without specifying any `delay` values. See the +**NOTE** in **Defining the Tests** above. + +If any of the `netem` keys inside of the `test_cases.yaml` are listed but empty this will +result in an invalid NetEm command and the test will fail. + +#### Assumptions + +In order to simplify parsing the YAML certain assumptions were made with regard to the +default units of NetEm commands. If these assumptions are incorrect an update may be +necessary. These assumptions are as follows: + +* The `min_delay` and `max_delay` parameters for `slot` are assumed to be in *μs* + +There is the option to include the units in the yaml (i.e. `slot: -800us`), but it would +making referencing that configuration value from a metric almost impossible. + +#### Options Not Supported + +NetEm options that require loading files are **not supported** at this time. + +### Metrics + +The `metrics` test case key contains a dictionary of named tests to apply to a given case. +Each entry is expected to be a Python expression that evaluates to a boolean value. If +the metric evaluates to `True`, that metric is considered to be met and will be +interpreted by PyTest as a PASS. An exception or falsely value is considered a failure. A +logical AND operation is applied across multiple metrics defined within a single test +case. + +The context that a metric expression is evaluated in contains a `results` dictionary that +is the loaded result of the JSON output from UDP-ST and a `test_case` dictionary that is +the loaded test case from `test_cases.yaml` + +An example of the `results` dictionary is shown here: + +```python +results = +{ + 'IPLayerMaxConnections': 1, + 'IPLayerMaxIncrementalResult': 3600, + 'IPLayerCapSupported': {'SoftwareVersion': '7.5.0', + 'ControlProtocolVersion': 9, + 'Metrics': 'IPLR,Sampled_RTT,IPDV,IPRR,RIPR'}, + 'Input': { + 'Interface': '', + 'Role': 'Receiver', + 'Host': 'server', + 'Port': 50639, + 'HostIPAddress': '172.23.0.2', + 'ClientIPAddress': '172.23.0.3', + 'ClientPort': 43557, + 'JumboFramesPermitted': 1, + 'NumberOfConnections': 1, + 'DSCP': 0, + 'ProtocolVersion': 'Any', + 'UDPPayloadMin': 48, + 'UDPPayloadMax': 8972, + 'UDPPayloadDefault': 1222, + 'UDPPayloadContent': 'zeroes', + 'TestType': 'Search', + 'IPDVEnable': 0, + 'IPRREnable': 1, + 'RIPREnable': 1, + 'PreambleDuration': 0, + 'StartSendingRateIndex': 0, + 'SendingRateIndex': -1, + 'NumberTestSubIntervals': 10, + 'NumberFirstModeTestSubIntervals': 0, + 'TestSubInterval': 1000, + 'StatusFeedbackInterval': 50, + 'TimeoutNoTestTraffic': 1000, + 'TimeoutNoStatusMessage': 1000, + 'Tmax': 1000, + 'TmaxRTT': 3000, + 'TimestampResolution': 1, + 'SeqErrThresh': 10, + 'ReordDupIgnoreEnable': 0, + 'LowerThresh': 30, + 'UpperThresh': 90, + 'HighSpeedDelta': 10, + 'SlowAdjThresh': 3, + 'HSpeedThresh': 1000000000, + 'RateAdjAlgorithm': 'B', + }, + 'Output': { + 'BOMTime': '2022-08-16T20:39:40.410973Z', + 'TmaxUsed': 1000, + 'TestInterval': 10, + 'TmaxRTTUsed': 3000, + 'TimestampResolutionUsed': 1, + 'Summary': { + 'DeliveredPercent': 100, + 'LossRatioSummary': 0, + 'ReorderedRatioSummary': 0, + 'ReplicatedRatioSummary': 0, + 'LossCount': 0, + 'ReorderedCount': 0, + 'ReplicatedCount': 0, + 'PDVMin': 0, + 'PDVAvg': 0, + 'PDVMax': 0.002, + 'PDVRangeSummary': 0.002, + 'RTTMin': 0, + 'RTTMax': 0.001, + 'RTTRangeSummary': 0.001, + 'IPLayerCapacitySummary': 1943.34, + 'InterfaceEthMbps': 0, + 'MinOnewayDelaySummary': 0, + 'MinRTTSummary': 0, + }, + 'AtMax': { + 'Mode': 1, + 'Intervals': 10, + 'TimeOfMax': '2022-08-16T20:39:50.472678Z', + 'DeliveredPercent': 100, + 'LossRatioAtMax': 0, + 'ReorderedRatioAtMax': 0, + 'ReplicatedRatioAtMax': 0, + 'LossCount': 0, + 'ReorderedCount': 0, + 'ReplicatedCount': 0, + 'PDVMin': 0, + 'PDVAvg': 0, + 'PDVMax': 0.002, + 'PDVRangeAtMax': 0.002, + 'RTTMin': 0, + 'RTTMax': 0.001, + 'RTTRangeAtMax': 0.001, + 'MaxIPLayerCapacity': 4114.99, + 'InterfaceEthMbps': 0, + 'MaxETHCapacityNoFCS': 4121.4, + 'MaxETHCapacityWithFCS': 4132.37, + 'MaxETHCapacityWithFCSVLAN': 4134.2, + 'MinOnewayDelayAtMax': 0, + }, + 'ModalResult': [], + 'EOMTime': '2022-08-16T20:39:50.923480Z', + 'Status': 'Complete', + }, + 'ErrorStatus': 0, + 'ErrorMessage': '', + } +``` + +An example of the `test_case` dictionary is shown here: + +```python +test_case = +{'case-2': { + 'client-cli': '-s -f json -d server', + 'server-cli': '-s -v', + 'netem': { + 'downstream': { + 'delay': { + 'params': [100]}, + 'rate': [1]}, + 'upstream': { + 'delay': { + 'params': [100]}, + 'rate': [1]}}, + 'metrics': { + 'reordered_lessthan_delivered': 'results["Output"]["Summary"]["DeliveredPercent"] / 100 > results["Output"]["Summary"]["ReorderedRatioSummary"]'}, + }} +``` + +The context that a metric expression is evaluated in provides access to any vanilla +Python functions, the Python [`math`](https://docs.python.org/3/library/math.html "Python math library doc page") +library, as well as two additional functions to evaluate if a result is within a +certain range or percent. + +These two additional functions have the following signatures: + +```python +def within_range(actual_value, min_val, max_val) + +def within_percent(actual_value, reference_value, percent_delta) +``` + +Some example metric expressions (which may not be specifically useful, but are valid) are +given here: + +```yaml +metrics: + within_good_pct: within_percent(results["Output"]["Summary"]["RTTMax"], 0.05, 10) + within_good_rng: within_range(results["Output"]["Summary"]["LossCount"], test_case[list(test_case.keys()[0])]["netem"]["downstream"]["corrupt"][0] - 10, test_case[list(test_case.keys()[0])]["netem"]["downstream"]["corrupt"][0] + 10) + loss_lessthan_10: results["Output"]["Summary"]["DeliveredPercent"] > 90.0 + range_not_nan: not math.isnan(results["Output"]["Summary"]["PDVRangeSummary"]) +``` + +## Running the Tests + +Run `pytest` from the `udpst\tests` directory. + +To generate the junit XML output required for bamboo and other task runners, the +command line flag `--junitxml=output.xml` can be used. diff --git a/tests/ci-test.sh b/tests/ci-test.sh new file mode 100644 index 00000000000000..3355606ecd2586 --- /dev/null +++ b/tests/ci-test.sh @@ -0,0 +1,3 @@ +#!/bin/sh +$NETEM_COMMAND || exit +/app/udpst $UDPST_COMMAND \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000000000..bd438908ab9014 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,56 @@ +# +# Copyright (c) 2022, Broadband Forum +# Copyright (c) 2022, UNH-IOL Communications +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# +# UDP Speed Test CI Testing - conftest.py +# +# This file provides the top level configuration and parameterization +# required by pytest, such as parsing the test-cases.yaml, and similar +# top level functionality. +# +# Author Date Comments +# -------------------- ---------- ---------------------------------- +# UNH-IOL Team 08/19/2022 Initial creation of the CI testing + +import yaml + +def pytest_addoption(parser): + parser.addoption( + "--udpstTestCasesYaml", action="store", default="test_cases.yaml", help="Filename of YAML test cases/" + ) + +def parameterIdent(testCase): + return next(iter(testCase)) + +def pytest_generate_tests(metafunc): + if 'testCase' in metafunc.fixturenames: + with open(metafunc.config.getoption('udpstTestCasesYaml'), 'r') as f: + yamlTestCases = yaml.load(f, Loader=yaml.FullLoader) + metafunc.parametrize('testCase', yamlTestCases, ids=parameterIdent) diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml new file mode 100644 index 00000000000000..1281bebceb7dec --- /dev/null +++ b/tests/docker-compose.yml @@ -0,0 +1,26 @@ +version: "3.3" # optional since v1.27.0 +services: + server: + build: + context: ../. + dockerfile: tests/Dockerfile + cap_add: + - NET_ADMIN + environment: + - MIN_TESTPORT=40000 + - MAX_TESTPORT=40005 + - NETEM_COMMAND=${DOWN_NETEM_COMMAND} + - UDPST_COMMAND=${SERVER_ARGS} + client: + build: + context: ../. + dockerfile: tests/Dockerfile + cap_add: + - NET_ADMIN + environment: + - MIN_TESTPORT=40000 + - MAX_TESTPORT=40005 + - NETEM_COMMAND=${UP_NETEM_COMMAND} + - UDPST_COMMAND=${CLIENT_ARGS} + depends_on: + - server diff --git a/tests/netem_parser.py b/tests/netem_parser.py new file mode 100644 index 00000000000000..b5955098114012 --- /dev/null +++ b/tests/netem_parser.py @@ -0,0 +1,96 @@ +""" +Takes in a dictionary and produces a netem command string for both the upstream (client to server) +and downstream (server to client) configurations. + +inputs: + yamlDict: a dictionary that was created from a udpst test yaml file +outputs: + upstreamNetemStr: a string representing a netem command for the upstream configuration + downstreamNetemStr: a string representing a netem command for the downstream configuration + +NOTE: There is no syntax checking to ensure the netem command string created is a valid command +string. It is the responsibility of the individual creating the test cases to ensure the yaml +follows the right schema. Future work might employ schema validation prior to this function +being called. +""" +from types import NoneType + + +def getNetemOpts(case): + if 'netem' in case.keys(): + netemOpts = case['netem'] + else: + return ['', ''] + if netemOpts is None: + return ['', ''] + + if 'upstream' in netemOpts.keys(): + upstream = netemOpts['upstream'] + upstreamNetemStr = netemEntry(upstream) + else: + upstreamNetemStr = '' + + if 'downstream' in netemOpts.keys(): + downstream = netemOpts['downstream'] + downstreamNetemStr = netemEntry(downstream) + else: + downstreamNetemStr = '' + + return [upstreamNetemStr, downstreamNetemStr] + +""" +Starts the netem command string concatenation + +inputs: + val: a dictionary that was created from a udpst test yaml file that contains either + upstream or downstream configurations +output: a string representing a netem command for the configuration +""" +def netemEntry(val): + return 'netem' + parse(val) + + +""" +Concatenates strings to create the remainder of the netem command string + +inputs: + val: a dictionary, list, or primitive type used to create the netem command string +output: a string representing a netem command for the configuration +""" +def parse(val): + if type(val) is dict: + return parseDict(val) + elif type(val) is list: + return parseList(val) + else: + return ' ' + str(val) + +""" +Concatenates the entries from a dictionary to create the remainder of the netem command string + +inputs: + val: a dictionary used to create a portion of the netem command string +output: a string representing a portion of a netem command for the configuration +""" +def parseDict(value): + silent_keys = ['params'] + output = '' + for (key, val) in value.items(): + if key in silent_keys: + output += parse(val) + else: + output += ' ' + key + parse(val) + return output + +""" +Concatenates the entries from a list to create the remainder of the netem command string + +inputs: + val: a list used to create a portion of the netem command string +output: a string representing a portion of a netem command for the configuration +""" +def parseList(value): + output = '' + for val in value: + output += parse(val) + return output diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 00000000000000..8f79baa008daaf --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,3 @@ +pytest_check == 1.0.*, >= 1.0.5 +pytest == 7.1.*, >= 7.1.2 +pyyaml == 6.*, >= 6.0 \ No newline at end of file diff --git a/tests/setup.sh b/tests/setup.sh new file mode 100755 index 00000000000000..9ac3c6f76340dd --- /dev/null +++ b/tests/setup.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +python3 -m venv ./ + +source ./bin/activate + +pip install -r requirements.txt + +docker compose build diff --git a/tests/test_cases.yaml b/tests/test_cases.yaml new file mode 100644 index 00000000000000..66a4c438633286 --- /dev/null +++ b/tests/test_cases.yaml @@ -0,0 +1,33 @@ +--- +- check-max-performance: + client-cli: "-s -f jsonf -d server" + server-cli: "-v -s -1" + metrics: + check-max-rate: results["Output"]["AtMax"]["MaxETHCapacityNoFCS"] > 2000 +- check-downstream-ds1Gbps-us100Mbps: + client-cli: "-s -f jsonf -d server" + server-cli: "-v -s -1" + netem: + downstream: + rate: + - 1000Mbit + upstream: + rate: + - 100Mbit + metrics: + downstream-summary-rate: within_range(results["Output"]["Summary"]["IPLayerCapacitySummary"], 700, 800) + downstream-max-rate: results["Output"]["AtMax"]["MaxETHCapacityNoFCS"] > 950 +- check-upstream-ds1Gbps-us100Mbps: + client-cli: "-s -f jsonf -u server" + server-cli: "-v -s -1" + netem: + downstream: + rate: + - 1000Mbit + upstream: + rate: + - 100Mbit + metrics: + upstream-summary-rate: within_range(results["Output"]["Summary"]["IPLayerCapacitySummary"], 90, 100) + upstream-max-rate: results["Output"]["AtMax"]["MaxETHCapacityNoFCS"] >= 95 +... diff --git a/tests/test_cases_sample.yaml b/tests/test_cases_sample.yaml new file mode 100644 index 00000000000000..3c85b45a8759ca --- /dev/null +++ b/tests/test_cases_sample.yaml @@ -0,0 +1,211 @@ +--- +- case-1: + client-cli: "client cli command options" + server-cli: "server cli command options" + netem: # full set of netem options, will not be a valid netem command + downstream: + limit: 1 # in packets + delay: + params: + - 2 # in ms + - 3 # in ms, jitter (optional) + - 4 # in percent, correlation (optional) + distribution: normal # { uniform | normal | pareto | paretonormal } (optional) + reorder: + params: + - 5 # in percent (optional) + - 6 # in percent, correlation (optional) + gap: 7 # as an integer (optional) + loss: + params: ecn # flag to use Explicit Congestion Notification + random: 8 # in percent + state: + - 9 # in percent, p13 + - 10 # in percent, p31 (optional) + - 11 # in percent, p32 (optional) + - 12 # in percent, p23 (optional) + - 13 # in percent, p14 (optional) + gemodel: + - 14 # in percent, p + - 15 # in percent, r (optional) + - 16 # in percent, 1-h (optional) + - 17 # in percent, 1-k (optional) + corrupt: + - 18 # in percent + - 19 # in percent, correlation (optional) + duplicate: + - 20 # in percent + - 21 # in percent, correlation (optional) + rate: + - 22 # in bits per second + - 23 # in bytes (can be negative), packetoverhead (optional) + - 24 # as an unsigned integer, cellsize (optional) + - 25 # as an integer (can be negative), celloverhead (optional) + slot: + params: + - 26 # assume default is in us, min_delay + - 27 # assume default is in us, max_delay, (optional) + distribution: + - normal # { uniform | normal | pareto | paretonormal | custom} (optional) + - 28 # in ms, delay + - 29 # in ms, jitter + packets: 30 # in packets, (optional) + bytes: 31 # in bytes, (optional) + upstream: + limit: 10 # in packets + delay: + params: + - 100 # in ms + - 10 # in ms, jitter (optional) + - 1 # in percent, correlation (optional) + distribution: normal # { uniform | normal | pareto | paretonormal } (optional) + reorder: + params: + - 25 # in percent + - 75 # in percent, correlation (optional) + gap: 4 # as an integer (optional) + loss: + params: ecn # flag to use Explicit Congestion Notification + random: 0.1 # in percent + state: + - 1 # in percent, p13 + - 5 # in percent, p31 (optional) + - 10 # in percent, p32 (optional) + - 50 # in percent, p23 (optional) + - 2 # in percent, p14 (optional) + gemodel: + - 1 # in percent, p + - 5 # in percent, r (optional) + - 10 # in percent, 1-h (optional) + - 10 # in percent, 1-k (optional) + corrupt: + - 1 # in percent + - 3 # in percent, correlation (optional) + duplicate: + - 1 # in percent + - 3 # in percent, correlation (optional) + rate: + - 1 # in bits per second + - 2 # in bytes (can be negative), packetoverhead (optional) + - 3 # as an unsigned integer, cellsize (optional) + - 5 # as an integer (can be negative), celloverhead (optional) + slot: + params: + - 800 # assume default is in us, min_delay + - 1000 # assume default is in us, max_delay, (optional) + distribution: + - normal # { uniform | normal | pareto | paretonormal | custom} (optional) + - 100 # in ms, delay + - 10 # in ms, jitter + packets: 10 # in packets, (optional) + bytes: 3 # in bytes, (optional) + metrics: + loss_lessthan_10: results["Output"]["Summary"]["DeliveredPercent"] > 90.0 + rtt_range_consistent: results["Output"]["Summary"]["RTTMax"] - results["Output"]["Summary"]["RTTMin"] < 1.0 +- case-2: + client-cli: "client cli command options round 2" + server-cli: "server cli command options round 2" + netem: # valid netem command will be produced + downstream: + delay: + params: + - 100 # in ms + rate: + - 1 # assume default is in Mbits + upstream: + delay: + params: + - 100 # in ms + rate: + - 1 # assume default is in Mbits + metrics: + reordered_lessthan_delivered: results["Output"]["Summary"]["DeliveredPercent"] / 100 > results["Output"]["Summary"]["ReorderedRatioSummary"] +- case-3: # will produce a valid netem command, numbers may not be sensical + client-cli: "client cli command options" + server-cli: "server cli command options" + netem: + downstream: + limit: 1 # in packets + delay: + params: + - 2 # in ms + - 3 # in ms, jitter (optional) + - 4 # in percent, correlation (optional) + distribution: normal # { uniform | normal | pareto | paretonormal } (optional) + reorder: + params: + - 5 # in percent (optional) + - 6 # in percent, correlation (optional) + gap: 7 # as an integer (optional) + loss: + state: + - 9 # in percent, p13 + - 10 # in percent, p31 (optional) + - 11 # in percent, p32 (optional) + - 12 # in percent, p23 (optional) + - 13 # in percent, p14 (optional) + corrupt: + - 18 # in percent + - 19 # in percent, correlation (optional) + duplicate: + - 20 # in percent + - 21 # in percent, correlation (optional) + rate: + - 22 # assume default is in Mbits + - 23 # in bytes (can be negative), packetoverhead (optional) + - 24 # as an unsigned integer, cellsize (optional) + - 25 # as an integer (can be negative), celloverhead (optional) + slot: + params: + - 26 # assume default is in us, min_delay + - 27 # assume default is in us, max_delay, (optional) + packets: 30 # in packets, (optional) + bytes: 31 # in bytes, (optional) + upstream: + limit: 10 # in packets + delay: + params: + - 100 # in ms + - 10 # in ms, jitter (optional) + - 1 # in percent, correlation (optional) + distribution: normal # { uniform | normal | pareto | paretonormal } (optional) + reorder: + params: + - 25 # in percent + - 75 # in percent, correlation (optional) + gap: 4 # as an integer (optional) + loss: + params: ecn # flag to use Explicit Congestion Notification + gemodel: + - 1 # in percent, p + - 5 # in percent, r (optional) + - 10 # in percent, 1-h (optional) + - 10 # in percent, 1-k (optional) + corrupt: + - 1 # in percent + - 3 # in percent, correlation (optional) + duplicate: + - 1 # in percent + - 3 # in percent, correlation (optional) + rate: + - 1 # assume default is in Mbits + - 2 # in bytes (can be negative), packetoverhead (optional) + - 3 # as an unsigned integer, cellsize (optional) + - 5 # as an integer (can be negative), celloverhead (optional) + slot: + distribution: + - normal # { uniform | normal | pareto | paretonormal | custom} (optional) + - 100 # in ms, delay + - 10 # in ms, jitter + packets: 10 # in packets, (optional) + bytes: 3 # in bytes, (optional) +- case-4: # example where netem key is not listed + client-cli: "client cli command options" + server-cli: "server cli command options" +- case-5: + client-cli: "client cli command options" + server-cli: "server cli command options" + netem: # example where netem key is provided but only downstream is listed (upstream not listed) + downstream: + limit: 1 # in packets +... diff --git a/tests/test_udpst.py b/tests/test_udpst.py new file mode 100644 index 00000000000000..44739de3a734f7 --- /dev/null +++ b/tests/test_udpst.py @@ -0,0 +1,166 @@ +# +# Copyright (c) 2022, Broadband Forum +# Copyright (c) 2022, UNH-IOL Communications +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# +# UDP Speed Test CI Testing - conftest.py +# +# This file provides the functional test cases and supporting functions +# for running test cases. The entry point as test_updst(testCase) is +# called once for each test case. +# +# Author Date Comments +# -------------------- ---------- ---------------------------------- +# UNH-IOL Team 08/19/2022 Initial creation of the CI testing + +import re +import netem_parser as np +import subprocess +import inspect +import json +import os + +from pytest_check import check_func +import traceback +import pytest_check as check + +import math + + +""" +Test Case Iterator + +Is called once by PyTest for each test case defined within the yaml +configuration file. This function handless the following actions. + +1. Parse the configuration into two valid NetEm commands for Us/Ds. +2. Use docker compose to run the test case, and collects the client output +3. Calls check_all_in_case() to evaluate all test metrics for the case. + +Note, any raised exception, due to parsing json output or failed metrics, +will cause the test case to fail and PyTest to move on to the next test +case in the yaml configuration. +""" +def test_udpst(testCase): + print('Test case was: ', testCase ) + + # Strip the test cast label from the object + testCase = testCase[list(testCase.keys())[0]] + + # Parse config to generate NetEm Command Line + netemOpts = np.getNetemOpts(testCase) # 1: upstream, 2: downstream + (upstreamOpts, downstreamOpts) = (netemOpts[0], netemOpts[1]) + + #results = gather_results(upstreamOpts, downstreamOpts, testCase["client-cli"], testCase["server-cli"]) + #def gather_results(upNetem, downNetem, clientArgs, serverArgs): + + # Setup the environment for docker compose, to pass in the arguments to the + # containers for test. + if upstreamOpts != '': + os.environ["UP_NETEM_COMMAND"] = 'tc qdisc replace dev eth0 root ' + upstreamOpts + else: + os.environ["UP_NETEM_COMMAND"] = '' + if downstreamOpts != '': + os.environ["DOWN_NETEM_COMMAND"] = 'tc qdisc replace dev eth0 root ' + downstreamOpts + else: + os.environ["DOWN_NETEM_COMMAND"] = '' + + os.environ["SERVER_ARGS"] = testCase["server-cli"] + os.environ["CLIENT_ARGS"] = testCase["client-cli"] + + # Clean up any hanging previous test runs (i.e. old containers) + subprocess.Popen(["docker", "compose", "--project-name", "udpst-testing", "down"]).wait() + subprocess.Popen(["docker", "compose", "--project-name", "udpst-testing", "rm", "-f"]).wait() + + # Run the test. + subprocess.Popen(["docker", "compose", "--project-name", "udpst-testing", "up", "--abort-on-container-exit"]).wait() + + # Grab the logs from the client container that contain the json output + results = subprocess.Popen(["docker", "logs", "udpst-testing-client-1"], stderr=subprocess.PIPE, stdout=subprocess.PIPE) + results.wait() + err = results.stderr.read().decode() + out = results.stdout.read().decode() + if err != '': + raise Exception("Container failed to run with: " + err) + elif out == '': + raise Exception("Container provided no output!") + else: + try: + results = json.loads(out) + except Exception as e: + raise e + + # Parse and check all metrics + check_all_in_case(testCase, results) + + +""" +Iterate over all metrics within the test case, and evaluate their result. + +test_case: dict following input case schema (plus metrics) +container_output: results json object from udpst clinet container +""" +def check_all_in_case(test_case, container_output): + metrics = test_case.get('metrics', {}) + if metrics is None: + return + for (metric_name, metric) in metrics.items(): + try: + result = eval( + metric, {}, {"test_case": test_case, "results": container_output, "within_range": within_range, "within_percent": within_percent, "math": math}) + + check.is_true(result, metric_name + ' : ' + metric) + + # try turning result into a bool (expr of test_text should have a bool as eval result) + except SyntaxError: + tb = traceback.format_exc() + check.is_true( + False, f"test metric was syntactically malformed: {metric_name}, traceback:\n{tb}") + except (ValueError, KeyError): + tb = traceback.format_exc() + check.is_true( + False, f"test metric referred to a key or variable that did not exist: {metric_name}, traceback:\n{tb}") + except Exception: + tb = traceback.format_exc() + check.is_true( + False, f"test metric caused an unexpected exception: {metric_name}, traceback:\n{tb}") + + +""" +Check value is within the range least to most, inclusively. +""" +def within_range(value, least, most): + return value >= least and value <= most + + +""" +Check if the actual value is within a plus/minus delta of a reference value. +""" +def within_percent(actual_value, reference_value, percent_delta): + return math.fabs((actual_value - reference_value) / reference_value) <= percent_delta / 100.0