diff --git a/.github/workflows/plot-interop.py b/.github/workflows/plot-interop.py new file mode 100644 index 00000000..e0877970 --- /dev/null +++ b/.github/workflows/plot-interop.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 + +import os +import sys + +import json +import matplotlib.pyplot as plt +import numpy as np +from matplotlib.colors import ListedColormap + +# QUIC implementes +CLIENT_IMPLS = ["tquic", "lsquic", "quiche", "picoquic", "ngtcp2", "msquic", + "s2n-quic", "quinn", "neqo", "kwik", "aioquic", "chrome", + "go-x-net", "quic-go", "mvfst"] + +SERVER_IMPLS = ["lsquic", "quiche", "picoquic", "ngtcp2", "msquic", "s2n-quic", + "quinn", "neqo", "kwik", "aioquic", "nginx", "haproxy", + "go-x-net", "quic-go", "mvfst"] + +# Interop test cases +INTEROP_TESTS = ["handshake", "retry", "resumption", "zerortt", "amplificationlimit", + "http3", "ipv6", "transfer", "multiplexing", "longrtt", "blackhole", + "handshakeloss", "handshakecorruption", "transferloss","transfercorruption"] + + +# Read a interop file generated by interop testing +def read_data(data_dir, server, client): + dirname = "%s-%s/%s-%s-logs" % (server, client, server, client) + path = os.path.join(data_dir, dirname, "interop.json") + try: + with open(path) as f: + data = json.load(f) + return convert_data(data["results"][0]) + except: + return [0] * len(INTEROP_TESTS) + + +# Convert interop results to a vector +def convert_data(result): + data = [0] * len(INTEROP_TESTS) + for i, name in enumerate(INTEROP_TESTS): + for item in result: + if item["name"] != name: + continue + if item["result"] == "succeeded": + data[i] = 3 + elif item["result"] == "unsupported": + data[i] = 2 + else: + data[i] = 1 + break + return data + + +# Convert test result to a letter +def convert_text(value): + if value == 3: + return "Y" # success + elif value == 2: + return "U" # unsupported + elif value == 1: + return "N" # failure + else: + return "-" # unknown + + +# Plot the interop graph +def plot(data_dir, is_tquic_server): + impls = CLIENT_IMPLS if is_tquic_server else SERVER_IMPLS + name = "server" if is_tquic_server else "client" + if is_tquic_server: + data = [read_data(data_dir, "tquic", impl) for impl in impls] + else: + data = [read_data(data_dir, impl, "tquic") for impl in impls] + interop_result = np.array(data) + print(interop_result) + + fig, ax = plt.subplots() + im = ax.imshow(interop_result, cmap=ListedColormap(['gray', 'red', 'blue', 'green']), + interpolation='nearest') + ax.set_xticks(np.arange(len(INTEROP_TESTS)), labels=INTEROP_TESTS) + ax.set_yticks(np.arange(len(impls)), labels=impls) + plt.setp(ax.get_xticklabels(), rotation=45, ha="right", + rotation_mode="anchor") + for i in range(len(INTEROP_TESTS)): + for j in range(len(impls)): + text = ax.text(j, i, convert_text(interop_result[i, j]), + ha="center", va="center", color="w") + ax.set_title("TQUIC %s interop results" % (name)) + fig.tight_layout() + + filename = "interop-tquic-%s.png" % (name) + plt.savefig(filename, dpi=300) + + +if __name__ == '__main__': + if len(sys.argv) < 2: + print("Usage: %s [data_dir]" % (sys.argv[0])) + exit(1) + + data_dir= sys.argv[1] + plot(data_dir, True) + plot(data_dir, False) + diff --git a/.github/workflows/tquic-interop-all.yml b/.github/workflows/tquic-interop-all.yml index 88af0f36..507f92e7 100644 --- a/.github/workflows/tquic-interop-all.yml +++ b/.github/workflows/tquic-interop-all.yml @@ -113,7 +113,7 @@ jobs: python3 -m json.tool ${{ matrix.server }}-${{ matrix.client }}-logs/interop.json - name: Store interop logs - if: ${{ failure() }} + if: ${{ always() }} uses: actions/upload-artifact@v4 with: name: ${{ matrix.server }}-${{ matrix.client }} @@ -121,3 +121,38 @@ jobs: quic-interop-runner/*logs/* !quic-interop-runner/*logs/**/crosstraffic/ !quic-interop-runner/*logs/**/goodput/ + + result: + runs-on: ubuntu-latest + needs: tquic_interop_testing + if: always() + steps: + - name: Download all workflow run artifacts + uses: actions/download-artifact@v4 + + - name: Display structure of downloaded files + run: ls -R + + - name: Display failed interop tests + run: grep -Ho "{[^{]*failed" */*/*.json + + - name: Download plot tools + uses: actions/checkout@v4 + with: + path: tools + + - name: Install dependences + run: | + sudo apt install python3-matplotlib + + - name: Plot all interop results + run: python3 tools/.github/workflows/plot-interop.py ./ + + - name: Store all interop results + uses: actions/upload-artifact@v4 + with: + name: tquic-interop-all-result + path: | + interop*.png + */*logs/* + !*/*logs/**/**/sim/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c374d3d..df54f946 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,7 +88,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - tquic_client: output the stats first and then exit when it receives an SIGINT signal. ### Changed -- Simplify FFI quic_set_logger() to avoid from return unnessary errors +- Simplify FFI quic_set_logger() to avoid from return unnecessary errors - Rename set_multipath() in Config to enable_multipath() - Rename set_multipath_algor() in Config to set_multipath_algorithm() - Change default congestion control algorithm to BBR diff --git a/typos.toml b/typos.toml index 0391977e..aa41b7cd 100644 --- a/typos.toml +++ b/typos.toml @@ -9,6 +9,10 @@ StatuS = "StatuS" [default.extend-words] # ECN Capable Transport ect = "ect" +# Packet Number +pn = "pn" +# Retransmission Timeout +RTO = "RTO" [files] extend-exclude = [