From dbd6b98dd14f1ca7d63b72f05b41ab2ff6acc544 Mon Sep 17 00:00:00 2001
From: Jake Shadle <jake.shadle@embark-studios.com>
Date: Tue, 28 Jan 2025 10:32:56 +0100
Subject: [PATCH] Add XDP integration test (#1078)

---
 .ci/xdp/cluster-config.yaml                | 15 ++++
 .ci/xdp/integration-test.sh                | 82 ++++++++++++++++++++++
 .ci/xdp/proxy.dockerfile                   |  3 +
 .ci/xdp/proxy.dockerfile.dockerignore      |  2 +
 .ci/xdp/server.yaml                        | 13 ++++
 .ci/xdp/spinup-cluster-with-registry.sh    | 47 +++++++++++++
 .ci/xdp/veth-integ-test.sh                 | 63 +++++++++++++++++
 .github/workflows/xdp-integration-test.yml | 46 ++++++++++++
 .gitignore                                 |  1 +
 9 files changed, 272 insertions(+)
 create mode 100644 .ci/xdp/cluster-config.yaml
 create mode 100755 .ci/xdp/integration-test.sh
 create mode 100644 .ci/xdp/proxy.dockerfile
 create mode 100644 .ci/xdp/proxy.dockerfile.dockerignore
 create mode 100644 .ci/xdp/server.yaml
 create mode 100755 .ci/xdp/spinup-cluster-with-registry.sh
 create mode 100755 .ci/xdp/veth-integ-test.sh
 create mode 100644 .github/workflows/xdp-integration-test.yml

diff --git a/.ci/xdp/cluster-config.yaml b/.ci/xdp/cluster-config.yaml
new file mode 100644
index 000000000..58ff3b24b
--- /dev/null
+++ b/.ci/xdp/cluster-config.yaml
@@ -0,0 +1,15 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+  # Proxy
+  - role: worker
+  # Client
+  - role: worker
+  # Server
+  - role: worker
+# https://github.com/helm/kind-action?tab=readme-ov-file#configuring-local-registry
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry]
+    config_path = "/etc/containerd/certs.d"
\ No newline at end of file
diff --git a/.ci/xdp/integration-test.sh b/.ci/xdp/integration-test.sh
new file mode 100755
index 000000000..5868958c5
--- /dev/null
+++ b/.ci/xdp/integration-test.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+set -eu
+
+source="${BASH_SOURCE[0]}"
+
+proxy_image="localhost:$REGISTRY_PORT/quilkin:ci"
+
+echo "::notice file=$source,line=$LINENO::Building quilkin proxy"
+cargo build -p quilkin --bin quilkin
+# strip the binary to reduce copy times into the dockerfile
+strip ./target/debug/quilkin
+
+echo "::notice file=$source,line=$LINENO::Building quilkin image"
+docker build -f .ci/xdp/proxy.dockerfile -t "${proxy_image}" .
+echo "::notice file=$source,line=$LINENO::Pushing quilkin image"
+docker push "${proxy_image}"
+
+echo "::notice file=$source,line=$LINENO::Starting UDP echo server"
+kubectl apply --context "$CLUSTER" -f .ci/xdp/server.yaml
+kubectl wait --context "$CLUSTER" --for=condition=ready pod/echo-server
+
+server_ip=$(kubectl get --context "$CLUSTER" pod echo-server --template '{{.status.podIP}}')
+
+echo "::notice file=$source,line=$LINENO::Starting quilkin proxy"
+kubectl apply --context "$CLUSTER" -f - <<EOF
+apiVersion: v1
+kind: Pod
+metadata:
+  name: proxy
+spec:
+  containers:
+  - name: proxy
+    image: "$proxy_image"
+    ports:
+    - containerPort: 7777
+      hostPort: 7777
+    args: ["proxy", "--to", "${server_ip}:8078", "--service.udp.xdp"]
+    securityContext:
+      capabilities:
+        add:
+        - CAP_BPF # We load an eBPF program
+        - CAP_NET_RAW # We create SOCK_RAW (XDP) sockets
+EOF
+
+kubectl wait --context "$CLUSTER" --for=condition=ready pod/proxy
+
+proxy_ip=$(kubectl get --context "$CLUSTER" pod proxy --template '{{.status.podIP}}')
+
+echo "::notice file=$source,line=$LINENO::Running UDP client"
+kubectl apply --context "$CLUSTER" -f - <<EOF
+apiVersion: v1
+kind: Job
+metadata:
+name: client
+spec:
+  containers:
+  - name: client
+    image: fortio/fortio
+    args: ["load", "-n", "100", "udp://${proxy_ip}:7777"]
+EOF
+
+logs=$(kubectl logs -f --context "$CLUSTER" job/client)
+
+echo "::notice file=$source,line=$LINENO::Finished sending client requests"
+
+# Total Bytes sent: 30, received: 0
+regex="Total Bytes sent: (\d+), received: (\d+)"
+
+if [[ $logs =~ $regex ]]; then
+  send=${BASH_REMATCH[1]}
+  recv=${BASH_REMATCH[2]}
+  if [[ $send -eq $recv ]]; then
+    echo "::notice file=$source,line=$LINENO::Successfully sent and received {$recv}b"
+    exit 0
+  fi
+
+  echo "::error file=$source,line=$LINENO::sent ${send}b but only received ${recv}b"
+  exit 1
+fi
+
+echo "::error file=$source,line=$LINENO::Failed to find expected log line from UDP client"
+exit 2
diff --git a/.ci/xdp/proxy.dockerfile b/.ci/xdp/proxy.dockerfile
new file mode 100644
index 000000000..5755588d0
--- /dev/null
+++ b/.ci/xdp/proxy.dockerfile
@@ -0,0 +1,3 @@
+FROM debian:bookworm-slim
+COPY ./target/debug/quilkin /usr/local/bin
+ENTRYPOINT quilkin
diff --git a/.ci/xdp/proxy.dockerfile.dockerignore b/.ci/xdp/proxy.dockerfile.dockerignore
new file mode 100644
index 000000000..6503f2ac8
--- /dev/null
+++ b/.ci/xdp/proxy.dockerfile.dockerignore
@@ -0,0 +1,2 @@
+# This file exists purely to ignore the root .dockerignore otherwise it will ignore
+# target/debug/quilkin when copying it into the image
diff --git a/.ci/xdp/server.yaml b/.ci/xdp/server.yaml
new file mode 100644
index 000000000..9fa61a8d5
--- /dev/null
+++ b/.ci/xdp/server.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: echo-server
+spec:
+  containers:
+  - name: echo-server
+    image: fortio/fortio
+    ports:
+    - containerPort: 8078
+      hostPort: 8078
+    args: ["udp-echo"]
+
diff --git a/.ci/xdp/spinup-cluster-with-registry.sh b/.ci/xdp/spinup-cluster-with-registry.sh
new file mode 100755
index 000000000..95966bc42
--- /dev/null
+++ b/.ci/xdp/spinup-cluster-with-registry.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+set -eu
+
+# 1. Create registry container unless it already exists
+if [ "$(docker inspect -f '{{.State.Running}}' "${CLUSTER_DESIRED}" 2>/dev/null || true)" != 'true' ]; then
+  docker run \
+    -d --restart=always -p "127.0.0.1:${REGISTRY_PORT}:5000" --network bridge --name "${REGISTRY_NAME}" \
+    registry:2
+fi
+
+kind create cluster --name "${CLUSTER_DESIRED}" --config=.ci/xdp/cluster-config.yaml
+
+# 3. Add the registry config to the nodes
+#
+# This is necessary because localhost resolves to loopback addresses that are
+# network-namespace local.
+# In other words: localhost in the container is not localhost on the host.
+#
+# We want a consistent name that works from both ends, so we tell containerd to
+# alias localhost:${reg_port} to the registry container when pulling images
+REGISTRY_DIR="/etc/containerd/certs.d/localhost:${REGISTRY_PORT}"
+for node in $(kind get nodes); do
+  docker exec "${node}" mkdir -p "${REGISTRY_DIR}"
+  cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${REGISTRY_DIR}/hosts.toml"
+[host."http://${REGISTRY_NAME}:5000"]
+EOF
+done
+
+# 4. Connect the registry to the cluster network if not already connected
+# This allows kind to bootstrap the network but ensures they're on the same network
+if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${REGISTRY_NAME}")" = 'null' ]; then
+  docker network connect "kind" "${REGISTRY_NAME}"
+fi
+
+# 5. Document the local registry
+# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "localhost:${REGISTRY_PORT}"
+    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
+EOF
diff --git a/.ci/xdp/veth-integ-test.sh b/.ci/xdp/veth-integ-test.sh
new file mode 100755
index 000000000..818c7bcfa
--- /dev/null
+++ b/.ci/xdp/veth-integ-test.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+set -e
+
+source="${BASH_SOURCE[0]}"
+
+cleanup() {
+    echo "Cleaning up"
+    ip netns del cs || true
+    ip netns del proxy || true
+
+    pkill fortio || true
+    pkill quilkin || true
+}
+
+trap cleanup EXIT
+
+ip netns del cs || true
+ip netns del proxy || true
+
+echo "::notice file=$source,line=$LINENO::Creating network namespaces"
+ip netns add cs
+ip netns add proxy
+
+echo "::notice file=$source,line=$LINENO::Adding client <-> proxy <-> server links"
+ip link add veth-cs type veth peer name veth-proxy
+
+ip link set veth-cs netns cs
+ip link set veth-proxy netns proxy
+
+echo "::notice file=$source,line=$LINENO::Adding IPs"
+ip -n cs addr add 10.0.0.1/24 dev veth-cs
+ip -n proxy addr add 10.0.0.2/24 dev veth-proxy
+
+echo "::notice file=$source,line=$LINENO::Creating network namespaces"
+ip -n cs link set veth-cs up
+ip -n proxy link set veth-proxy up
+
+ip netns exec cs fortio udp-echo&
+ip netns exec proxy ./target/debug/quilkin proxy --to 10.0.0.1:8078 --publish.udp.xdp&
+
+echo "::notice file=$source,line=$LINENO::Launching client"
+ip netns exec cs fortio load -n 10 udp://10.0.0.2:7777 2> ./target/logs.txt
+logs=$(cat ./target/logs.txt)
+echo "$logs"
+
+regex="Total Bytes sent: ([0-9]+), received: ([0-9]+)"
+
+if [[ $logs =~ $regex ]]; then
+  send=${BASH_REMATCH[1]}
+  recv=${BASH_REMATCH[2]}
+  # We could be more strict here and require they are exactly equal, but I can't
+  # even consistently get that on my local machine so I doubt CI will fair better
+  if [[ $recv -ne "0" ]]; then
+    echo "::notice file=$source,line=$LINENO::Successfully sent ${send}B and received ${recv}B"
+    exit 0
+  fi
+
+  echo "::error file=$source,line=$LINENO::sent ${send}B but only received ${recv}B"
+  exit 1
+fi
+
+echo "::error file=$source,line=$LINENO::Failed to find expected log line from UDP client"
+exit 2
diff --git a/.github/workflows/xdp-integration-test.yml b/.github/workflows/xdp-integration-test.yml
new file mode 100644
index 000000000..99f788fe2
--- /dev/null
+++ b/.github/workflows/xdp-integration-test.yml
@@ -0,0 +1,46 @@
+name: xdp-integration-test
+
+on:
+  push:
+    branches:
+      - "main"
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  xdp-integration:
+    runs-on: ubuntu-24.04
+    # env:
+    #   CLUSTER_DESIRED: xdp-integ
+    #   CLUSTER: kind-xdp-integ
+    #   SERVER_PORT: "8078"
+    #   REGISTRY_PORT: "5001"
+    #   REGISTRY_NAME: registry
+    steps:
+      - uses: actions/checkout@v4
+      # - name: Create cluster
+      #   id: kind
+      #   uses: helm/kind-action@v1
+      #   with:
+      #     config: .ci/xdp/cluster-config.yaml
+      #     cluster_name: ${{ env.CLUSTER_DESIRED }}
+      #     registry: true
+      #     registry_name: ${{ env.REGISTRY_NAME }}
+      #     registry_port: ${{ env.REGISTRY_PORT }}
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Fetch quilkin
+        run: cargo fetch --target x86_64-unknown-linux-gnu
+      - name: Build quilkin
+        run: cargo build -p quilkin --bin quilkin
+      - name: Install fortio
+        run: |
+          curl -L -o fortio.deb https://github.com/fortio/fortio/releases/download/v1.68.0/fortio_1.68.0_amd64.deb
+          sudo dpkg -i fortio.deb
+      - name: Run XDP integration test
+        run: sudo .ci/xdp/veth-integ-test.sh
diff --git a/.gitignore b/.gitignore
index d4bfdd3e3..d9815e46c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,6 +19,7 @@
 !.gcloudignore
 !.dockerignore
 !.github
+!.ci
 !xds/ci/.golangci.yaml
 
 *.iml