Update tquic-interop-all.yml and add plot tools for interop testing (#…

…211)

.github/workflows/plot-interop.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+
+import json
+import matplotlib.pyplot as plt
+import numpy as np
+from matplotlib.colors import ListedColormap
+
+# QUIC implementes
+CLIENT_IMPLS = ["tquic", "lsquic", "quiche", "picoquic", "ngtcp2", "msquic",
+                "s2n-quic", "quinn", "neqo", "kwik", "aioquic", "chrome",
+                "go-x-net", "quic-go", "mvfst"]
+
+SERVER_IMPLS = ["lsquic", "quiche", "picoquic", "ngtcp2", "msquic", "s2n-quic",
+                "quinn", "neqo", "kwik", "aioquic", "nginx", "haproxy",
+                "go-x-net", "quic-go", "mvfst"]
+
+# Interop test cases
+INTEROP_TESTS = ["handshake", "retry", "resumption", "zerortt", "amplificationlimit",
+                 "http3", "ipv6", "transfer", "multiplexing", "longrtt", "blackhole", 
+                 "handshakeloss", "handshakecorruption", "transferloss","transfercorruption"]
+
+
+# Read a interop file generated by interop testing
+def read_data(data_dir, server, client):
+    dirname = "%s-%s/%s-%s-logs" % (server, client, server, client)
+    path = os.path.join(data_dir, dirname, "interop.json")
+    try:
+        with open(path) as f:
+            data = json.load(f)
+            return convert_data(data["results"][0])
+    except:
+        return [0] * len(INTEROP_TESTS)
+
+
+# Convert interop results to a vector
+def convert_data(result):
+    data = [0] * len(INTEROP_TESTS)
+    for i, name in enumerate(INTEROP_TESTS):
+        for item in result:
+            if item["name"] != name:
+                continue
+            if item["result"] == "succeeded":
+                data[i] = 3
+            elif item["result"] == "unsupported":
+                data[i] = 2
+            else:
+                data[i] = 1
+            break
+    return data
+
+
+# Convert test result to a letter
+def convert_text(value):
+    if value == 3:
+       return "Y" # success
+    elif value == 2:
+       return "U" # unsupported
+    elif value == 1:
+       return "N" # failure
+    else:
+       return "-" # unknown
+
+
+# Plot the interop graph
+def plot(data_dir, is_tquic_server):
+    impls = CLIENT_IMPLS if is_tquic_server else SERVER_IMPLS
+    name = "server" if is_tquic_server else "client"
+    if is_tquic_server:
+        data = [read_data(data_dir, "tquic", impl) for impl in impls]
+    else:
+        data = [read_data(data_dir, impl, "tquic") for impl in impls]
+    interop_result = np.array(data)
+    print(interop_result)
+
+    fig, ax = plt.subplots()
+    im = ax.imshow(interop_result, cmap=ListedColormap(['gray', 'red', 'blue', 'green']),
+                   interpolation='nearest')
+    ax.set_xticks(np.arange(len(INTEROP_TESTS)), labels=INTEROP_TESTS)
+    ax.set_yticks(np.arange(len(impls)), labels=impls)
+    plt.setp(ax.get_xticklabels(), rotation=45, ha="right",
+             rotation_mode="anchor")
+    for i in range(len(INTEROP_TESTS)):
+        for j in range(len(impls)):
+            text = ax.text(j, i, convert_text(interop_result[i, j]),
+                           ha="center", va="center", color="w")
+    ax.set_title("TQUIC %s interop results" % (name))
+    fig.tight_layout()
+
+    filename = "interop-tquic-%s.png" % (name)
+    plt.savefig(filename, dpi=300)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print("Usage: %s [data_dir]" % (sys.argv[0]))
+        exit(1)
+
+    data_dir= sys.argv[1]
+    plot(data_dir, True)
+    plot(data_dir, False)
+

.github/workflows/tquic-interop-all.yml

@@ -113,11 +113,46 @@ jobs:
         python3 -m json.tool ${{ matrix.server }}-${{ matrix.client }}-logs/interop.json
 
     - name: Store interop logs
-      if: ${{ failure() }}
+      if: ${{ always() }}
       uses: actions/upload-artifact@v4
       with:
         name: ${{ matrix.server }}-${{ matrix.client }}
         path: |
           quic-interop-runner/*logs/*
           !quic-interop-runner/*logs/**/crosstraffic/
           !quic-interop-runner/*logs/**/goodput/
+
+  result:
+    runs-on: ubuntu-latest
+    needs: tquic_interop_testing
+    if: always()
+    steps:
+    - name: Download all workflow run artifacts
+      uses: actions/download-artifact@v4
+
+    - name: Display structure of downloaded files
+      run: ls -R
+
+    - name: Display failed interop tests
+      run: grep -Ho "{[^{]*failed" */*/*.json
+
+    - name: Download plot tools
+      uses: actions/checkout@v4
+      with:
+        path: tools
+
+    - name: Install dependences
+      run: |
+        sudo apt install python3-matplotlib
+
+    - name: Plot all interop results
+      run: python3 tools/.github/workflows/plot-interop.py ./
+
+    - name: Store all interop results
+      uses: actions/upload-artifact@v4
+      with:
+        name: tquic-interop-all-result
+        path: |
+          interop*.png
+          */*logs/*
+          !*/*logs/**/**/sim/

CHANGELOG.md

@@ -88,7 +88,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - tquic_client: output the stats first and then exit when it receives an SIGINT signal.
 
 ### Changed
-- Simplify FFI quic_set_logger() to avoid from return unnessary errors
+- Simplify FFI quic_set_logger() to avoid from return unnecessary errors
 - Rename set_multipath() in Config to enable_multipath()
 - Rename set_multipath_algor() in Config to set_multipath_algorithm()
 - Change default congestion control algorithm to BBR

typos.toml

@@ -9,6 +9,10 @@ StatuS = "StatuS"
 [default.extend-words]
 # ECN Capable Transport
 ect = "ect"
+# Packet Number
+pn = "pn"
+# Retransmission Timeout
+RTO = "RTO"
 
 [files]
 extend-exclude = [